From bf818655fbb8a4a7903b3112b6dda1114ae0da3d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 5 Apr 2024 13:01:58 +0200 Subject: [PATCH 1/4] Move -ci- jobs to out of the index processing Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 62 +++++++++++++++++------------ 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index af5097a43..019d37053 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -324,38 +324,23 @@ def ciconf_per_lib_index = [:].withDefault { [:] } def pkgconf_per_src_index = [:].withDefault { [:] } generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index) -// Generate PR jobs: 1 per ci configuration on each lib -ciconf_per_lib_index.each { lib_name, lib_configs -> - lib_configs.each { ci_configs -> - def config_name = ci_configs.getKey() +gz_collections_yaml.collections.each { collection -> + collection.ci.configs.each { config_name -> def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } - def branches_with_collections = ci_configs.getValue() - def branch_names = branches_with_collections.collect { it.branch }.unique() - def script_name_prefix = cleanup_library_name(lib_name) - def gz_job_name_prefix = lib_name.replaceAll('-','_') def distro = ci_config.system.version def arch = ci_config.system.arch def categories_enabled = ci_config.ci_categories_enabled - def ws_checkout_dir = lib_name - if (ci_config.exclude.all?.contains(lib_name)) - return - assert(lib_name) - assert(branch_names) - assert(ci_config) - assert(categories_enabled) + collection.libs.each { lib -> + def lib_name = lib.name + def branch_name = lib.repo.current_branch + def gz_job_name_prefix = lib_name.replaceAll('-','_') - // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) - branches_with_collections.each { branch_and_collection -> - def gz_ci_job - branch_name = branch_and_collection.branch - if (categories_enabled.contains('stable_branches')) - { + if (categories_enabled.contains('stable_branches')) { if (ci_config.system.so == 'linux') { gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) if (categories_enabled.contains('stable_branches_asan')) { - // Generate asan jobs on Linux def gz_ci_asan_job = job("${gz_job_name_prefix}-ci_asan-${branch_name}-${distro}-${arch}") generate_asan_ci_job(gz_ci_asan_job, lib_name, branch_name, ci_config) gz_ci_asan_job.with @@ -364,8 +349,9 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> scm(Globals.CRON_ON_WEEKEND) } } + logging_list['asan_ci'].add( - [collection: branch_and_collection.collection, + [collection: collection.name, job_name: gz_ci_asan_job.name]) } } else if (ci_config.system.so == 'darwin') { @@ -389,10 +375,34 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> } logging_list['branch_ci'].add( - [collection: branch_and_collection.collection, + [collection: collection.name, job_name: gz_ci_job.name]) } // end of daily category enabled - } // end_of_branch + } + } +} + +return + +// Generate PR jobs: 1 per ci configuration on each lib +ciconf_per_lib_index.each { lib_name, lib_configs -> + lib_configs.each { ci_configs -> + def config_name = ci_configs.getKey() + def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } + def branches_with_collections = ci_configs.getValue() + def branch_names = branches_with_collections.collect { it.branch }.unique() + def script_name_prefix = cleanup_library_name(lib_name) + def gz_job_name_prefix = lib_name.replaceAll('-','_') + def distro = ci_config.system.version + def arch = ci_config.system.arch + def categories_enabled = ci_config.ci_categories_enabled + def ws_checkout_dir = lib_name + if (ci_config.exclude.all?.contains(lib_name)) + return + assert(lib_name) + assert(branch_names) + assert(ci_config) + assert(categories_enabled) if (categories_enabled.contains('pr')) { @@ -429,7 +439,7 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> } // end of ci_any_job if (categories_enabled.contains('stable_branches') && \ - (! ci_config.exclude.abichecker?.contains(lib_name))) + (! ci_config.exclude.abichecker?.contains(lib_name))) { // ABI branch jobs (-ci-abichecker-) for non main branches def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}" From c112ed5872e84ca129bc19a471ca663911d07d1d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 15 May 2024 09:38:56 +0200 Subject: [PATCH 2/4] WIP: PR implementation Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 019d37053..516e4a1c1 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -379,6 +379,22 @@ gz_collections_yaml.collections.each { collection -> job_name: gz_ci_job.name]) } // end of daily category enabled } + + if (categories_enabled.contains('pr')) + { + if (ci_config.system.so == 'linux') + { + def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') + def extra_cmd = pre_setup_script ?: "" + if (categories_enabled.contains('stable_branches') && \ + (! ci_config.exclude.abichecker?.contains(lib_name))) + { + // generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements) + } + } else if (ci_config.system.so == 'darwin') { + } else if (ci_config.system.so == 'windows') { + } + } } From d4af6e156383fb36d14bd1de63d25e4a19492819 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 15 May 2024 17:04:46 +0200 Subject: [PATCH 3/4] Replace indexes by looping over directly to respect ci_config Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 184 +++++++++++++--------------- 1 file changed, 87 insertions(+), 97 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 516e4a1c1..b48270ad5 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -323,7 +323,13 @@ def generate_debbuilder_job(src_name, ArrayList pre_setup_script_hooks) def ciconf_per_lib_index = [:].withDefault { [:] } def pkgconf_per_src_index = [:].withDefault { [:] } generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index) - +/* + * + * Loop over each collection, inside each collection loop over the ci configurations assigned + * and finally loop over each collection library listed. + * + */ +def branch_index = [:].withDefault { [:] } gz_collections_yaml.collections.each { collection -> collection.ci.configs.each { config_name -> def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } @@ -335,6 +341,32 @@ gz_collections_yaml.collections.each { collection -> def branch_name = lib.repo.current_branch def gz_job_name_prefix = lib_name.replaceAll('-','_') + // Build the branch_index while going through all the libraries to avoid + // looping twice. + if (ci_config.system.so == 'linux') { + platform = distro + } else if (ci_config.system.so == 'darwin') { + platform = 'homebrew' + } else if (ci_config.system.so == 'windows') { + platform = 'windows' + } + branch_index[lib_name][platform] = branch_index[lib_name][platform]?: ['pr':[], 'pr_abichecker':[]] + if (! ci_config.exclude.all?.contains(lib_name)) + { + if (categories_enabled.contains('pr')) + { + branch_index[lib_name][platform]['pr'].contains(branch_name) ?: + branch_index[lib_name][platform]['pr'] << [branch: branch_name, ci_name: config_name] + } + if (categories_enabled.contains('pr_abichecker') && + (! ci_config.exclude.abichecker?.contains(lib_name))) + { + branch_index[lib_name][platform]['pr_abichecker'].contains(branch_name) ?: + branch_index[lib_name][platform]['pr_abichecker'] << [branch: branch_name, ci_name: config_name] + } + } + + // Generate jobs for the library entry being parsed if (categories_enabled.contains('stable_branches')) { if (ci_config.system.so == 'linux') { gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") @@ -379,54 +411,25 @@ gz_collections_yaml.collections.each { collection -> job_name: gz_ci_job.name]) } // end of daily category enabled } - - if (categories_enabled.contains('pr')) - { - if (ci_config.system.so == 'linux') - { - def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') - def extra_cmd = pre_setup_script ?: "" - if (categories_enabled.contains('stable_branches') && \ - (! ci_config.exclude.abichecker?.contains(lib_name))) - { - // generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements) - } - } else if (ci_config.system.so == 'darwin') { - } else if (ci_config.system.so == 'windows') { - } - } } -return - -// Generate PR jobs: 1 per ci configuration on each lib -ciconf_per_lib_index.each { lib_name, lib_configs -> - lib_configs.each { ci_configs -> - def config_name = ci_configs.getKey() - def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } - def branches_with_collections = ci_configs.getValue() - def branch_names = branches_with_collections.collect { it.branch }.unique() - def script_name_prefix = cleanup_library_name(lib_name) - def gz_job_name_prefix = lib_name.replaceAll('-','_') - def distro = ci_config.system.version - def arch = ci_config.system.arch - def categories_enabled = ci_config.ci_categories_enabled - def ws_checkout_dir = lib_name - if (ci_config.exclude.all?.contains(lib_name)) - return - assert(lib_name) - assert(branch_names) - assert(ci_config) - assert(categories_enabled) - - if (categories_enabled.contains('pr')) - { +branch_index.each { lib_name, distro_configs -> + distro_configs.each { distro, branches_with_ciconfig -> + if (branches_with_ciconfig['pr']) { + def branch_names = branches_with_ciconfig['pr'].collect { it.branch }.unique() + // Hack that assumes that pre_setup_script_hook, arch and requirements are equal on all + // ciconfigs associated to a given branch and distro combination + def config_name = branches_with_ciconfig['pr'].collect { it.ci_name }.unique()[0] + def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } + def script_name_prefix = cleanup_library_name(lib_name) + def gz_job_name_prefix = lib_name.replaceAll('-','_') + def arch = ci_config.system.arch + def ws_checkout_dir = lib_name if (ci_config.system.so == 'linux') { def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') def extra_cmd = pre_setup_script ?: "" - def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" def gz_ci_any_job = job(gz_ci_job_name) OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, @@ -435,57 +438,6 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> ENABLE_CPPCHECK, branch_names) generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements) - gz_ci_any_job.with - { - steps - { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - - ${GLOBAL_SHELL_CMD} - ${extra_cmd} - - export BUILDING_SOFTWARE_DIRECTORY=${lib_name} - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/${script_name_prefix}-compilation.bash - """.stripIndent()) - } // end of steps - } // end of ci_any_job - - if (categories_enabled.contains('stable_branches') && \ - (! ci_config.exclude.abichecker?.contains(lib_name))) - { - // ABI branch jobs (-ci-abichecker-) for non main branches - def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}" - def abi_job = job(abi_job_name) - OSRFLinuxABIGitHub.create(abi_job) - GenericAnyJobGitHub.create(abi_job, - "gazebosim/${lib_name}", - branch_names - [ 'main']) - generate_label_by_requirements(abi_job, lib_name, ci_config.requirements) - abi_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - - ${GLOBAL_SHELL_CMD} - ${extra_cmd} - - export ARCH=${arch} - export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} - export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch} - export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl} - export ABI_JOB_SOFTWARE_NAME=${lib_name} - /bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash - """.stripIndent()) - } // end of steps - } // end of with - } } else if (ci_config.system.so == 'darwin') { // -------------------------------------------------------------- def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64" @@ -510,9 +462,48 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> add_win_devel_bat_call(gz_win_ci_any_job, lib_name, ws_checkout_dir) Globals.gazebodistro_branch = false } - } //end of pr enabled - } //end of lib_configs -} // end of lib + } + + if (branches_with_ciconfig['pr_abichecker']) { + def branch_names = branches_with_ciconfig['pr_abichecker'].collect { it.branch }.unique() + // Hack that assumes that pre_setup_script_hook, arch and requirements are equal on all + // ciconfigs associated to a given branch and distro combination + def config_name = branches_with_ciconfig['pr_abichecker'].collect { it.ci_name }.unique()[0] + def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } + def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') + def extra_cmd = pre_setup_script ?: "" + def arch = ci_config.system.arch + def gz_job_name_prefix = lib_name.replaceAll('-','_') + def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}" + def abi_job = job(abi_job_name) + OSRFLinuxABIGitHub.create(abi_job) + GenericAnyJobGitHub.create(abi_job, + "gazebosim/${lib_name}", + branch_names - [ 'main']) + generate_label_by_requirements(abi_job, lib_name, ci_config.requirements) + abi_job.with + { + steps { + shell("""\ + #!/bin/bash -xe + + export DISTRO=${distro} + + ${GLOBAL_SHELL_CMD} + ${extra_cmd} + + export ARCH=${arch} + export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} + export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch} + export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl} + export ABI_JOB_SOFTWARE_NAME=${lib_name} + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash + """.stripIndent()) + } // end of steps + } // end of with + } + } +} pkgconf_per_src_index.each { pkg_src, pkg_src_configs -> // For each entry in the index perform two steps: @@ -615,7 +606,6 @@ collection_job_names.each { collection_name, job_names -> name(jobname) } def collection = gz_collections_yaml.collections.find { it.name == collection_name } - println(collection) if (collection.packaging?.linux?.nightly) { collection.libs.each { lib -> name(get_debbuilder_name(lib, collection.packaging)) From a204368f46314b24e8d742f64a1b0265103f2009 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 15 May 2024 17:06:27 +0200 Subject: [PATCH 4/4] Remove duplicate abichecker Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/dsl_checks.bash | 20 ++++++------ jenkins-scripts/dsl/gz-collections.yaml | 41 +++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/jenkins-scripts/dsl/dsl_checks.bash b/jenkins-scripts/dsl/dsl_checks.bash index e4dfe8f25..a8ae04cd7 100755 --- a/jenkins-scripts/dsl/dsl_checks.bash +++ b/jenkins-scripts/dsl/dsl_checks.bash @@ -53,13 +53,13 @@ fi # re-enable after https://github.com/gazebo-tooling/release-tools/issues/1095 # Filter out the previous auto jobs -# filtered_dir=$(mktemp -d) -# cp -- *-abichecker-*.xml "${filtered_dir}" -# rm -f "${filtered_dir}"/*-ubuntu_auto*.xml -# repeated=$(grep '\' "${filtered_dir}"/*-abichecker-*.xml | awk '{ print $2 }' | sort | uniq -d) -# if [[ -n ${repeated} ]]; then -# echo "Found a duplicate in an abichecker branch:" -# echo "${repeated}" -# echo "please exclude one of the versions in the yaml file to reduce the server workload" -# exit 1 -# fi +filtered_dir=$(mktemp -d) +cp -- *-abichecker-*.xml "${filtered_dir}" +rm -f "${filtered_dir}"/*-ubuntu_auto*.xml +repeated=$(grep '\' "${filtered_dir}"/*-abichecker-*.xml | awk '{ print $2 }' | sort | uniq -d) +if [[ -n ${repeated} ]]; then + echo "Found a duplicate in an abichecker branch:" + echo "${repeated}" + echo "please exclude one of the versions in the yaml file to reduce the server workload" + exit 1 +fi diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 3dc9dc5fc..bd993b2f9 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -1,4 +1,4 @@ -# + # SPEC: not yet approved, used as draft # #--- @@ -256,7 +256,7 @@ collections: current_branch: main ci: configs: - - focal + - focal_exclude_harmonic_abi_dups - brew - win packaging: @@ -502,6 +502,43 @@ ci_configs: - pr_abichecker - stable_branches - stable_branches_asan + - name: focal_exclude_harmonic_abi_dups + system: + so: linux + distribution: ubuntu + version: focal + arch: amd64 + requirements: + large_memory: + - gz-physics + nvidia_gpu: + - gz-sim + - gz-gui + - gz-rendering + - gz-sensors + exclude: + all: + - gz-citadel + - gz-fortress + - gz-garden + abichecker: + - gz-cmake + - gz-tools + # Exclude the abichecker that are duplicate with Harmonic + # The ones on Harmonic run on jammy + - gz-common + - gz-math + - gz-plugin + - gz-utils + pre_setup_script_hook: + gz-physics: + - "export MAKE_JOBS=1" + tests_disabled: + ci_categories_enabled: + - pr + - pr_abichecker + - stable_branches + - stable_branches_asan - name: brew system: so: darwin