Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add diegorusso-aarch64-bigmem worker #546

Merged
merged 4 commits into from
Oct 29, 2024
Merged

Add diegorusso-aarch64-bigmem worker #546

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5c4e84b
Add diegorusso-aarch64-bigmem worker
diegorusso Oct 24, 2024
8b9ee92
Address feedback
diegorusso Oct 25, 2024
25c1339
Improve code to delay the build in case it falls in a time range
diegorusso Oct 28, 2024
d75d799
Decrease numbers of parallel tests
diegorusso Oct 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions master/custom/builders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
FedoraRawhideFreedthreadingBuild,
UnixAsanBuild,
UnixAsanDebugBuild,
UnixBigmemBuild,
UnixTraceRefsBuild,
UnixVintageParserBuild,
UnixRefleakBuild,
Expand Down Expand Up @@ -240,6 +241,8 @@
("PPC64LE CentOS9 LTO + PGO", "cstratak-CentOS9-ppc64le", LTOPGONonDebugBuild),

# Linux aarch64 GCC/Clang
("aarch64 Ubuntu 22.04 BigMem", "diegorusso-aarch64-bigmem", UnixBigmemBuild),

# Fedora Rawhide is unstable
("aarch64 Fedora Rawhide", "cstratak-fedora-rawhide-aarch64", FedoraRawhideBuild),
("aarch64 Fedora Rawhide Refleaks", "cstratak-fedora-rawhide-aarch64", UnixRefleakBuild),
Expand Down
6 changes: 6 additions & 0 deletions master/custom/factories.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,12 @@ class UnixBuildWithoutDocStrings(UnixBuild):
configureFlags = ["--with-pydebug", "--without-doc-strings"]


class UnixBigmemBuild(UnixBuild):
buildersuffix = ".bigmem"
testFlags = ["-M60g", "-j4", "-uall,extralargefile"]
factory_tags = ["bigmem"]


class AIXBuild(UnixBuild):
configureFlags = [
"--with-pydebug",
Expand Down
6 changes: 6 additions & 0 deletions master/custom/workers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,12 @@ def get_workers(settings):
tags=['linux', 'unix', 'rhel', 'arm', 'arm64', 'aarch64'],
parallel_tests=40,
),
cpw(
name="diegorusso-aarch64-bigmem",
tags=['linux', 'unix', 'ubuntu', 'arm', 'arm64', 'aarch64', 'bigmem'],
not_branches=['3.9', '3.10', '3.11', '3.12', '3.13'],
parallel_tests=60,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just noticing this value, it seems a little extreme for a bigmem builder. Theoretically, this could call on 3600GB of RAM (60 tests x 60GB each from the -M60g option). In practice there aren't enough bigmem tests to actually request that, but this is a big enough number of tests that it becomes likely that multiple (possibly all) bigmem tests will be running in parallel.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, my bad. Decreased to 4, so we should be good (60*4=240)

),
cpw(
name="cstratak-rhel8-s390x",
tags=['linux', 'unix', 'rhel', 's390x'],
Expand Down
73 changes: 52 additions & 21 deletions master/master.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import os
import subprocess
import sys

from datetime import timedelta
from datetime import datetime, timedelta
from functools import partial

from buildbot.plugins import reporters, schedulers, util
Expand Down Expand Up @@ -182,6 +182,24 @@ def is_important_change(change):
return any(is_important_file(filename) for filename in change.files)


# Avoid a build to be started between start and end time and delay such build
# at end time
def no_builds_between(start, end):
def f(builder, requests):
now = datetime.now()
if start <= now.hour < end:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This condition will never be true with start=22 and end=2.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right. I wrote the prototype but then left as is. I'll fix it tomorrow.

Copy link
Contributor Author

@diegorusso diegorusso Oct 28, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hopefully now the code is better to read and understand. Now it works when the time range spans over midnight.

# Calculate the delay until end time
delay = ((end - now.hour) % 24) * 3600 - now.minute * 60 - now.second
builder.master.reactor.callLater(
delay,
builder.buildset_manager.submitBuildSet,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're getting an exception here:

    Traceback (most recent call last):
      File "/srv/buildbot/venv/lib/python3.9/site-packages/twisted/internet/defer.py", line 980, in _startRunCallbacks
        self._runCallbacks()
      File "/srv/buildbot/venv/lib/python3.9/site-packages/twisted/internet/defer.py", line 1074, in _runCallbacks
        current.result = callback(  # type: ignore[misc]
      File "/srv/buildbot/venv/lib/python3.9/site-packages/twisted/internet/defer.py", line 1960, in _gotResultInlineCallbacks
        _inlineCallbacks(r, gen, status, context)
      File "/srv/buildbot/venv/lib/python3.9/site-packages/twisted/internet/defer.py", line 2014, in _inlineCallbacks
        result = context.run(gen.send, result)
    --- <exception caught here> ---
      File "/srv/buildbot/venv/lib/python3.9/site-packages/buildbot/process/buildrequestdistributor.py", line 262, in _getNextUnclaimedBuildRequest
        nextBreq = yield self.nextBuild(self.bldr, breqs)
      File "/srv/buildbot/master/master.cfg", line 216, in f
        builder.buildset_manager.submitBuildSet,
    builtins.AttributeError: 'Builder' object has no attribute 'buildset_manager'

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I'm reading the nextBuild and Twisted docs correctly, this should return a Deferred like this:

from twisted.internet import defer
...
            deferred = defer.Deferred()
            builder.master.reactor.callLater(
                int(delay),
                deferred.callback,
                requests[0],
            )
            return deferred

But, it would be nice to get confirmation from someone who's used Twisted before :)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I filed #553

requests[0],
)
return None # Do not start the build immediately
return requests[0] # Start the next build request
return f


github_status_builders = []
release_status_builders = []
mail_status_builders = []
Expand Down Expand Up @@ -247,17 +265,24 @@ for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches):
mail_status_builders.append(buildername)
github_status_builders.append(buildername)
release_status_builders.append(buildername)
c["builders"].append(
util.BuilderConfig(
name=buildername,
workernames=[worker_name],
builddir="%s.%s%s"
% (branchname, worker_name, getattr(f, "buildersuffix", "")),
factory=f,
tags=tags,
locks=[cpulock.access("counting")],
)

builder = util.BuilderConfig(
name=buildername,
workernames=[worker_name],
builddir="%s.%s%s"
% (branchname, worker_name, getattr(f, "buildersuffix", "")),
factory=f,
tags=tags,
locks=[cpulock.access("counting")],
)

# This worker runs pyperformance at 12am. If a build is scheduled between
# 10pm and 2am, it will be delayed at 2am.
if worker_name == "diegorusso-aarch64-bigmem":
builder.nextBuild = no_builds_between(22, 2)

c["builders"].append(builder)

c["schedulers"].append(
schedulers.SingleBranchScheduler(
name=branchname,
Expand Down Expand Up @@ -315,18 +340,24 @@ for name, worker_name, buildfactory, stability, tier in BUILDERS:
tags = ["PullRequest", stability, *getattr(f, "tags", [])]
if tier:
tags.append(tier)
c["builders"].append(
util.BuilderConfig(
name=buildername,
workernames=[worker_name],
builddir="%s.%s%s"
% ("pull_request", worker_name, getattr(f, "buildersuffix", "")),
factory=f,
tags=tags,
locks=[cpulock.access("counting")],
)

builder = util.BuilderConfig(
name=buildername,
workernames=[worker_name],
builddir="%s.%s%s"
% ("pull_request", worker_name, getattr(f, "buildersuffix", "")),
factory=f,
tags=tags,
locks=[cpulock.access("counting")],
)

# This worker runs pyperformance at 12am. If a build is scheduled between
# 10pm and 2am, it will be delayed at 2am.
if worker_name == "diegorusso-aarch64-bigmem":
builder.nextBuild = no_builds_between(22, 2)

c["builders"].append(builder)

c["schedulers"].append(
GitHubPrScheduler(
name="pull-request-scheduler",
Expand Down
Loading