master.cfg

# -*- python -*-
# ex: set filetype=python:

from buildbot.plugins import *

import os
import shlex
from datetime import timedelta

c = BuildmasterConfig = {}

import user_config

current_dir = os.path.dirname(os.path.abspath(__file__))

## SECRETS MANAGEMENT

secrets_dir = os.path.join(current_dir, "master-secrets")

c["secretsProviders"] = [secrets.SecretInAFile(dirname=secrets_dir)]

## LOG JANITORING

c["configurators"] = [
    util.JanitorConfigurator(
        logHorizon=timedelta(weeks=16),
        hour=12,
        dayOfWeek=6,
    )
]

## CHANGE SOURCES

# keep it empty here; we'll hook it from github via www
c["change_source"] = []
c["collapseRequests"] = True

## MASTER TO WORKER INTERFACE

# the master will listen here
c["protocols"] = {}
c["protocols"][user_config.master_protocol] = {"port": user_config.master_port}

## DATABASE

c["db"] = {
    "db_url": user_config.master_db,
}

## WEB INTERFACE AND IDENTITY

# allow specific users from github org
authz = util.Authz(
    allowRules=[util.AnyControlEndpointMatcher(role="buildbot")],
    roleMatchers=[util.RolesFromGroups(groupPrefix="chimera-linux/")],
)
if user_config.github_client_id:
    bauth = util.GitHubAuth(
        user_config.github_client_id,
        util.Secret("github-client-secret"),
        apiVersion=4,
        getTeamsMembership=True,
    )
else:
    bauth = None

c["title"] = user_config.title
c["titleURL"] = user_config.title_url

c["buildbotURL"] = user_config.bot_url

c["www"] = dict(
    port=user_config.www_port,
    authz=authz,
    auth=bauth,
    change_hook_dialects=dict(github={"secret": util.Secret("github-webhook-token")}),
    plugins=dict(
        waterfall_view={},
        console_view={},
        grid_view={},
    ),
)
## SERVICES

# TODO: irc bot
c["services"] = []

## MACHINE DEFINITION
#
# Here we conflate worker, scheduler, and builder into one, as there is one
# worker per architecture, one scheduler per worker, one builder per
# scheduler, in order to make sure all kick in upon changes being received

c["workers"] = []
c["schedulers"] = []
c["builders"] = []


@util.renderer
def sort_pkgs(props, cmd_base):
    ret = cmd_base + ["bulk-print"]
    # get packages list
    pkgs = str(props.getProperty("pkgs_unbuilt")).strip().split("\n")
    # map it
    ret += map(lambda v: v.split("=")[0], pkgs)
    # that's about it
    return ret


@util.renderer
def make_build(props, cmd_base):
    ret = []
    # list of all pkgs, sorted in bulk order
    pkgs = str(props.getProperty("pkgs_sorted")).split()
    # get the versioned list and make it into a mapping
    vermap = {}
    for pkg in str(props.getProperty("pkgs_unbuilt")).strip().split("\n"):
        pl = pkg.split("=")
        if len(pl) == 2:
            vermap[pl[0]] = pl[1]
    # make up shellargs
    for p in pkgs:
        ret.append(
            util.ShellArg(
                command=cmd_base + ["--force-color", "--stage", "pkg", p],
                logname=f"pkg:{p}={vermap[p] if p in vermap else 'unknown'}",
                haltOnFailure=True,
            )
        )
    # emit an unstage command if the list is non-empty
    if ret:
        ret.append(
            util.ShellArg(
                command=cmd_base + ["unstage"],
                logname="unstage",
                warnOnFailure=True,
            )
        )
    return ret


def gen_machine(c, machdict):
    archn = machdict["arch"]

    workn = f"worker-{archn}"
    schedn = f"scheduler-{archn}"
    buildn = f"builder-{archn}"

    # add worker for the arch
    w = worker.Worker(
        workn, util.Secret(f"pass-worker-{archn}"), properties={}, defaultProperties={}
    )
    c["workers"].append(w)

    # add scheduler for the arch
    s = schedulers.SingleBranchScheduler(
        name=schedn,
        change_filter=util.ChangeFilter(branch=user_config.cports_branch),
        # give the changes a chance to accumulate
        treeStableTimer=2,
        builderNames=[buildn],
    )
    c["schedulers"].append(s)

    # add force scheduler for the arch
    sf = schedulers.ForceScheduler(
        name=f"force-{schedn}",
        buttonName="boop",
        label="Poke the builder",
        reason=util.StringParameter(
            name="reason", label="Reason:", required=True, default="boop"
        ),
        builderNames=[buildn],
    )
    c["schedulers"].append(sf)

    # create build factory
    f = util.BuildFactory()

    cmd_base = ["./cbuild"]
    # only pass if specified
    if "config" in machdict:
        cmd_base += ["-c", machdict["config"]]
    if "opts" in machdict:
        cmd_base += machdict["opts"]

    rsync = [
        "rsync",
        "-amrt",
        "--progress",
        "--exclude",
        "*.lock",
        "-e",
        user_config.repo_ssh,
    ]
    rsync_dest = [
        machdict["repo-src"],
        f"{user_config.repo_dest}:{machdict['repo-dest']}",
    ]

    f.addStep(
        steps.Git(
            repourl=user_config.cports_repo,
            mode="incremental",
            alwaysUseLatest=True,
            name="cports_update",
            description="Updating cports",
            descriptionDone="Updated cports",
            logEnviron=False,
            haltOnFailure=True,
        )
    )
    f.addStep(
        steps.ShellCommand(
            command=cmd_base + ["bootstrap-update"],
            name="bldroot_update",
            description="Bldroot update",
            descriptionDone="Bldroot updated",
            logEnviron=False,
            haltOnFailure=True,
            env={"PYTHONUNBUFFERED": "1"},
        )
    )
    # unsorted, but versioned
    f.addStep(
        steps.SetPropertyFromCommand(
            command=cmd_base + ["list-unbuilt"],
            property="pkgs_unbuilt",
            name="find_unbuilt",
            description="Find unbuilt",
            descriptionDone="Found unbuilt",
            logEnviron=False,
            haltOnFailure=True,
            env={"PYTHONUNBUFFERED": "1"},
        )
    )
    # get a bulk-sorted, plain list
    f.addStep(
        steps.SetPropertyFromCommand(
            command=sort_pkgs.withArgs(cmd_base),
            property="pkgs_sorted",
            name="sort_unbuilt",
            description="Sort unbuilt",
            descriptionDone="Sorted unbuilt",
            logEnviron=False,
            haltOnFailure=True,
            env={"PYTHONUNBUFFERED": "1"},
        )
    )
    f.addStep(
        steps.ShellSequence(
            commands=make_build.withArgs(cmd_base),
            name="build_packages",
            description="Build packages",
            descriptionDone="Built packages",
            logEnviron=False,
            haltOnFailure=True,
            timeout=14400,
            env={"PYTHONUNBUFFERED": "1"},
        )
    )
    f.addStep(
        steps.ShellCommand(
            command=cmd_base + ["prune-pkgs"],
            name="prune_packages",
            description="Prune packages",
            descriptionDone="Pruned packages",
            logEnviron=False,
            haltOnFailure=True,
            env={"PYTHONUNBUFFERED": "1"},
        )
    )
    f.addStep(
        steps.ShellCommand(
            command=rsync
            + [
                "--exclude",
                "*.gz",
            ]
            + rsync_dest,
            name="upload_packages",
            description="Upload packages",
            descriptionDone="Uploaded packages",
            logEnviron=False,
            haltOnFailure=True,
        )
    )
    f.addStep(
        steps.ShellCommand(
            command=rsync
            + [
                "--delete",
            ]
            + rsync_dest,
            name="sync_repos",
            description="Synchronize repos",
            descriptionDone="Synchronized repos",
            logEnviron=False,
            haltOnFailure=True,
        )
    )
    if user_config.repo_post:
        f.addStep(
            steps.ShellCommand(
                command=shlex.split(user_config.repo_ssh)
                + [user_config.repo_dest]
                + user_config.repo_post,
                name=user_config.repo_hook,
                description=user_config.repo_hookdesc,
                descriptionDone=user_config.repo_hookdone,
                logEnviron=False,
                warnOnFailure=True,
            )
        )

    # add builder for the arch
    b = util.BuilderConfig(name=buildn, workernames=[workn], factory=f)
    c["builders"].append(b)


for machdict in user_config.machines:
    gen_machine(c, machdict)