diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 19d303df..68137b3f 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -2,6 +2,7 @@ # ex: set filetype=python: import os +from dataclasses import dataclass from buildbot.plugins import steps, util, worker from buildbot.steps.shell import Test @@ -9,6 +10,7 @@ from constants import BUILDERS_INSTALL, OS_INFO from master_common import base_master_config from utils import canStartBuild, envFromProperties, getScript, nextBuild, printEnv + cfg_dir = os.path.abspath(os.path.dirname(__file__)) # Non autogen master. For now the directory structure is: @@ -33,6 +35,51 @@ c = BuildmasterConfig = base_master_config(config) artifactsURL = os.environ["ARTIFACTS_URL"] +INSTALL_UPGRADE_PROPERTIES_LIST = [ + # Must be set to True in BB for upgrade scripts to work + "BB_CI", + # TODO(cvicentiu) compared to arm64 instead of aarch64 in scripts/deb-install.sh + # amd64, aarch64, ppc64le, s390x, x86 + "arch", + # Where artifacts are downloaded from ci.mariadb.org + "artifactsURL", + # Impacts upgrade and install logic. Same value as the branch in git + # (incl. bb- prefix and other suffixes). + "branch", + # almalinux, centos, debian, fedora, openeuler, rhel, ubuntu, rockylinux + "dist_name", + # compared to "yes" in rpm-upgrade.sh, but never set. + # TODO(cvicentiu) investigate and fix this. + "is_main_tree", + # TODO(cvicentiu) This should be handled within the script, not BB, mariadb_version should be enough. + # extracted from mariadb_version variable as a separate BB step. + "major_version", + # Eg. mariadb-10.11.12, comes from parent (tarball) builder props, + # based on tarball filename. + "mariadb_version", + # Set from parent (tarball) builder. First two numbers from mariadb_version. + "master_branch", + # TODO(cvicentiu) investigate and remove this one. + # Set to "yes" / "no" within this file, but never used anywhere. + "needsGalera", + # Set to the builder that triggered this install/upgrade + # Used during install scripts to create builder URL. + "parentbuildername", + # TODO(cvicentiu) This should be removed from install / upgrade scripts. + # and finally removed from here. + # Always set to "yes" within this file. + "systemdCapability", + # "server", "all", "columnstore", "deps" + # Impacts which tests are run. Set within this file only. + "test_mode", + # "major", "minor" + # Impacts which tests are run. set within this file. + "test_type", + # Loaded from os_info.yaml. Impacts upgrade & install tests. + "version_name", +] + + ####### UTILS def getRpmUpgradeStep(): return Test( @@ -40,25 +87,7 @@ def getRpmUpgradeStep(): haltOnFailure=True, description=["testing", "upgrade"], descriptionDone=["test", "upgrade"], - env=envFromProperties( - [ - "BB_CI", - "arch", - "artifactsURL", - "branch", - "dist_name", - "is_main_tree", - "major_version", - "mariadb_version", - "master_branch", - "needsGalera", - "parentbuildername", - "systemdCapability", - "test_mode", - "test_type", - "version_name", - ] - ), + env=envFromProperties(INSTALL_UPGRADE_PROPERTIES_LIST), command=["./rpm-upgrade.sh"], ) @@ -69,24 +98,7 @@ def getRpmInstallStep(): haltOnFailure=True, description=["testing", "install"], descriptionDone=["test", "install"], - env=envFromProperties( - [ - "BB_CI", - "arch", - "artifactsURL", - "branch", - "dist_name", - "major_version", - "mariadb_version", - "master_branch", - "needsGalera", - "parentbuildername", - "systemdCapability", - "test_mode", - "test_type", - "version_name", - ] - ), + env=envFromProperties(INSTALL_UPGRADE_PROPERTIES_LIST), command=["./rpm-install.sh"], ) @@ -97,24 +109,7 @@ def getDebUpgradeStep(): haltOnFailure=True, description=["testing", "upgrade"], descriptionDone=["test", "upgrade"], - env=envFromProperties( - [ - "BB_CI", - "arch", - "artifactsURL", - "branch", - "dist_name", - "major_version", - "mariadb_version", - "master_branch", - "needsGalera", - "parentbuildername", - "systemdCapability", - "test_mode", - "test_type", - "version_name", - ] - ), + env=envFromProperties(INSTALL_UPGRADE_PROPERTIES_LIST), command=["./deb-upgrade.sh"], ) @@ -125,24 +120,7 @@ def getDebInstallStep(): haltOnFailure=True, description=["testing", "install"], descriptionDone=["test", "install"], - env=envFromProperties( - [ - "BB_CI", - "arch", - "artifactsURL", - "branch", - "dist_name", - "major_version", - "mariadb_version", - "master_branch", - "needsGalera", - "parentbuildername", - "systemdCapability", - "test_mode", - "test_type", - "version_name", - ] - ), + env=envFromProperties(INSTALL_UPGRADE_PROPERTIES_LIST), command=["./deb-install.sh"], ) @@ -208,6 +186,23 @@ f_rpm_upgrade.addStep(getRpmUpgradeStep()) c["workers"] = [] c["builders"] = [] + +@dataclass +class LibVirtWorkerSpecs: + name: str + connection_url: str + image_path: str + password: str + + def __init__(self, platform: str, os_name: str, os_version: str): + host, url = config["private"]["libvirt_workers"][platform] + name = f"{host}-{os_name}-{os_version}-{platform}", url + super().__init__(name=name, + connection_url=url, + image_path=f"/var/libvirt/images/{name}", + password=config["private"]["worker_pass"]["libvirt"]) + + # Add the workers and builds based on the configured install builders (see constants.py) for builder_name in BUILDERS_INSTALL: # Parse builder name @@ -217,50 +212,28 @@ for builder_name in BUILDERS_INSTALL: os_info_name = os_name + "-" + os_version - libvirt_worker_name = ( - config["private"]["libvirt_workers"][platform][0] - + "-" - + os_name - + "-" - + os_version - + "-" - + platform - ) - connection_url = config["private"]["libvirt_workers"][platform][1] - image_path = "/var/libvirt/images/" + libvirt_worker_name + worker_specs = LibVirtWorkerSpecs(platform=platform, + os_name=os_name, + os_version=os_version) c["workers"].append( worker.LibVirtWorker( - libvirt_worker_name, - config["private"]["worker_pass"]["libvirt"], - util.Connection(connection_url), - image_path, + worker_specs.name, + worker_specs.password, + util.Connection(worker_specs.connection_url), + worker_specs.image_path, build_wait_timeout=0, max_builds=1, ) ) - if builder_type == "deb": - factory_install = f_deb_install - factory_upgrade = f_deb_upgrade - build_arch = platform - elif builder_type == "rpm": - factory_install = f_rpm_install - factory_upgrade = f_rpm_upgrade - build_arch = ( - os_name + str(OS_INFO[os_info_name]["version_name"]) + "-" + platform - ) - - # FIXME - all RPM's should follow the same conventions! - if os_name == "centos" and OS_INFO[os_info_name]["version_name"] >= 9: - if platform == "amd64": - platform = "x86_64" - build_arch = f"centos/{OS_INFO[os_info_name]['version_name']}/{platform}" + factory_install = f_deb_install if builder_type == "deb" else f_rpm_install + factory_upgrade = f_deb_upgrade if builder_type == "deb" else f_rpm_upgrade c["builders"].append( util.BuilderConfig( name=builder_name, - workernames=libvirt_worker_name, + workernames=worker_specs.name, tags=[os_name, builder_type, "install", "kvm"], collapseRequests=True, nextBuild=nextBuild, @@ -270,7 +243,7 @@ for builder_name in BUILDERS_INSTALL: "needsGalera": "yes", "dist_name": os_name, "version_name": OS_INFO[os_info_name]["version_name"], - "arch": build_arch, + "arch": platform, "BB_CI": True, "artifactsURL": artifactsURL, }, @@ -284,7 +257,7 @@ for builder_name in BUILDERS_INSTALL: c["builders"].append( util.BuilderConfig( name=major_upgrade_name, - workernames=libvirt_worker_name, + workernames=worker_specs.name, tags=[os_name, builder_type, "upgrade", "kvm", "major", "server"], collapseRequests=True, nextBuild=nextBuild, @@ -294,7 +267,7 @@ for builder_name in BUILDERS_INSTALL: "needsGalera": "yes", "dist_name": os_name, "version_name": OS_INFO[os_info_name]["version_name"], - "arch": build_arch, + "arch": platform, "test_mode": "server", "test_type": "major", "BB_CI": True, @@ -310,7 +283,7 @@ for builder_name in BUILDERS_INSTALL: c["builders"].append( util.BuilderConfig( name=minor_upgrade_name + "-all", - workernames=libvirt_worker_name, + workernames=worker_specs.name, tags=[os_name, builder_type, "upgrade", "kvm", "minor", "all"], collapseRequests=True, nextBuild=nextBuild, @@ -320,7 +293,7 @@ for builder_name in BUILDERS_INSTALL: "needsGalera": "yes", "dist_name": os_name, "version_name": OS_INFO[os_info_name]["version_name"], - "arch": build_arch, + "arch": platform, "test_mode": "all", "test_type": "minor", "BB_CI": True, @@ -334,7 +307,7 @@ for builder_name in BUILDERS_INSTALL: c["builders"].append( util.BuilderConfig( name=minor_upgrade_name + "-columnstore", - workernames=libvirt_worker_name, + workernames=worker_specs.name, tags=[os_name, builder_type, "upgrade", "kvm", "minor", "columnstore"], collapseRequests=True, nextBuild=nextBuild, @@ -344,7 +317,7 @@ for builder_name in BUILDERS_INSTALL: "needsGalera": "no", "dist_name": os_name, "version_name": OS_INFO[os_info_name]["version_name"], - "arch": build_arch, + "arch": platform, "test_mode": "columnstore", "test_type": "minor", "BB_CI": True, diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 2752440b..0a703f94 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -143,6 +143,68 @@ rpm_repo_dir() { set -u } +_rpm_get_base_mirror_url() { + local branch=$1 + echo "https://rpm.mariadb.org/$branch" +} + +_rpm_get_base_archive_url() { + local branch=$1 + echo "https://archive.mariadb.org/mariadb-$branch/yum/" +} + +_rpm_get_repo_path() { + # Construct a distro repo URL path based on the file hierarchy set up + # by buildbot / release scripts. + local arch=$1 + local dist_name=$2 + local version_number=$3 + local path="${dist_name}${version_number}-${arch}" + + # Special handling for centos 9 + # This uses the previous hierarchy in release scripts instead of a flattened + # path. + if [[ "$dist_name" == "centos" && "$version_number" -ge 9 ]]; then + # Check if architecture is amd64 and convert to x86_64 + if [[ "$arch" == "amd64" ]]; then + arch="x86_64" + fi + path="$dist_name/$version_number/$arch" + fi + + echo "$path" +} + +rpm_get_mirror_url() { + # Return full URL to corresponding distro repo on the mariadb mirror. + local branch=$1 + local arch=$2 + local dist_name=$3 + local version_number=$4 + local base + local path + + base=$(_rpm_get_base_mirror_url "$branch") + path=$(_rpm_get_repo_path "$arch" "$dist_name" "$version_number") + # Print the final constructed URL + echo "${base}/${path}" +} + +rpm_get_archive_url() { + # Return full URL to corresponding distro repo on the archive. + local branch=$1 + local arch=$2 + local dist_name=$3 + local version_number=$4 + local base + local path + + base=$(_rpm_get_base_archive_url "$branch") + path=$(_rpm_get_repo_path "$arch" "$dist_name" "$version_number") + # Print the final constructed URL + echo "${base}/${path}" +} + rpm_pkg() { # ID_LIKE may not exist set +u @@ -279,15 +341,25 @@ rpm_setup_mariadb_mirror() { bb_log_err "missing the branch variable" exit 1 } - branch=$1 + [[ -n $2 ]] || { + bb_log_err "missing the mirror_url variable" + exit 1 + } + [[ -n $3 ]] || { + bb_log_err "missing the archive_url variable" + exit 1 + } + local branch=$1 + local mirror_url=$2 + local archive_url=$3 + bb_log_info "setup MariaDB repository for $branch branch" command -v wget >/dev/null || { bb_log_err "wget command not found" exit 1 } - #//TEMP it's probably better to install the last stable release here...? - mirror_url="https://rpm.mariadb.org/$branch/$arch" - archive_url="https://archive.mariadb.org/mariadb-$branch/yum/$arch" + + local baseurl if wget -q --spider "$mirror_url"; then baseurl="$mirror_url" elif wget -q --spider "$archive_url"; then @@ -297,7 +369,7 @@ rpm_setup_mariadb_mirror() { # since we know it will always fail. But apparently, it's not going to # happen soon in BB. Once done though, replace the warning with an error # and use a non-zero exit code. - bb_log_warn "rpm_setup_mariadb_mirror: $branch packages for $dist_name $version_name does not exist on https://rpm.mariadb.org/" + bb_log_warn "rpm_setup_mariadb_mirror: $branch packages do not exist on either $mirror_url or $archive_url" exit 0 fi cat <