From 687c0d52f892645fddf2471bf904f5698a380c93 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Thu, 2 Mar 2017 18:31:04 -0500 Subject: [PATCH 1/8] RPMB-129 Fixed bug with removing build directories --- SOURCES/buildmon | 24 +++++++++++++----------- SOURCES/initenv | 29 +++++++++++++++++++++++++++++ SOURCES/rpmbuilder | 15 +++++++-------- rpmbuilder-node.spec | 9 ++++++++- rpmbuilder.spec | 5 ++++- 5 files changed, 61 insertions(+), 21 deletions(-) create mode 100755 SOURCES/initenv diff --git a/SOURCES/buildmon b/SOURCES/buildmon index 700f9ee..6e80ee4 100755 --- a/SOURCES/buildmon +++ b/SOURCES/buildmon @@ -3,7 +3,7 @@ ############################################################################### APP="RPMBuilder Node Monitor" -VER="1.1.0" +VER="1.1.1" ############################################################################### @@ -12,7 +12,7 @@ BUILDER_DIR="/home/builder" EXEC="$BUILDER_DIR/buildmon" BUILD_LOCK="$BUILDER_DIR/.buildlock" CLEAN_DELAY=600 -KEEP_DAYS=7 # 1 Week +MAX_KEEP_DAYS=7 # 1 Week DIR_LIST="BUILD BUILDROOT RPMS SOURCES SPECS SRPMS" CWD=$(pwd) @@ -21,15 +21,15 @@ CWD=$(pwd) main() { trap doExit SIGINT SIGTERM + check + doExit } check() { local has_lock - local dl - - dl=0 + local counter=0 while : ; do sleep 1 @@ -38,23 +38,25 @@ check() { touch $BUILD_LOCK has_lock=true else - if [[ $has_lock ]] ; then + if [[ -n $has_lock ]] ; then rm -f $BUILD_LOCK has_lock="" fi fi - ((dl++)) + ((counter++)) - if [[ $dl -eq $CLEAN_DELAY ]] ; then - dl=0 ; clean + if [[ $counter -eq $CLEAN_DELAY ]] ; then + counter=0 + clean fi done } clean() { - local rbdir="$BUILDER_DIR/rpmbuild" - find $rbdir -maxdepth 1 -type d -mtime +$KEEP_DAYS -delete + for dir in $DIR_LIST ; do + find "$BUILDER_DIR/rpmbuild/$dir" -maxdepth 1 -mtime +$MAX_KEEP_DAYS -delete + done } doExit() { diff --git a/SOURCES/initenv b/SOURCES/initenv new file mode 100755 index 0000000..efe7d3f --- /dev/null +++ b/SOURCES/initenv @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +############################################################################### + +APP="Build Env Init" +VER="1.0.0" + +############################################################################### + +BUILDER_HOME="/home/builder" + +############################################################################### + +main() { + initEnv &> /dev/null +} + +initEnv() { + mkdir -p "$BUILDER_HOME/rpmbuild/BUILD" + mkdir -p "$BUILDER_HOME/rpmbuild/BUILDROOT" + mkdir -p "$BUILDER_HOME/rpmbuild/RPMS" + mkdir -p "$BUILDER_HOME/rpmbuild/SOURCES" + mkdir -p "$BUILDER_HOME/rpmbuild/SPECS" + mkdir -p "$BUILDER_HOME/rpmbuild/SRPMS" +} + +############################################################################### + +main "$@" diff --git a/SOURCES/rpmbuilder b/SOURCES/rpmbuilder index add3913..2c2c2e5 100755 --- a/SOURCES/rpmbuilder +++ b/SOURCES/rpmbuilder @@ -1,7 +1,7 @@ #!/usr/bin/env bash APP="RPMBuilder" -VER="1.5.2" +VER="1.5.3" #################################################################### @@ -128,6 +128,8 @@ LONG_BUILD=300 BUILD_LOCK=".buildlock" +BUILDER_HOME="/home/builder" + BUILD_LOCK_LOCAL="/tmp/rpmbuilder-local-lock" BUILD_LOCK_PARALLEL="/tmp/rpmbuilder-parallel-lock" LOCK_TIME_MAX_DIFF=10800 # 3 Hours @@ -831,14 +833,11 @@ remoteCheckEnv() { doExit $ERROR_REMOTE fi - sshCommand "$user@$host" "ls rpmbuild 2>&1" &> /dev/null - - if [[ $? -ne 0 ]] ; then - show "rpmbuild directory structure not created on remote server." $RED - doExit $ERROR_REMOTE - fi + # Print info about node + sshCommand "$user@$host" "$BUILDER_HOME/nodeinfo 2>/dev/null" - sshCommand "$user@$host" "/home/builder/nodeinfo 2>/dev/null" + # Initialize build environment + sshCommand "$user@$host" "$BUILDER_HOME/initenv" &> /dev/null } remoteDepsInstall() { diff --git a/rpmbuilder-node.spec b/rpmbuilder-node.spec index 826c8d9..16d82bc 100644 --- a/rpmbuilder-node.spec +++ b/rpmbuilder-node.spec @@ -51,7 +51,7 @@ Summary: Configuration package for rpmbuilder node Name: rpmbuilder-node -Version: 1.2.2 +Version: 1.3.0 Release: 0%{?dist} License: EKOL Group: Development/Tools @@ -61,6 +61,7 @@ Source0: %{service_name} Source1: %{service_name}.init Source2: %{user_name}.sudoers Source3: nodeinfo +Source4: initenv Source10: rpmmacros_centos6 Source11: rpmmacros_centos7 @@ -97,6 +98,7 @@ install -pm 755 %{SOURCE0} %{buildroot}%{home_dir}/ install -pm 755 %{SOURCE1} %{buildroot}%{_initddir}/%{service_name} install -pm 644 %{SOURCE2} %{buildroot}%{_sysconfdir}/sudoers.d/%{user_name} install -pm 755 %{SOURCE3} %{buildroot}%{home_dir}/ +install -pm 755 %{SOURCE4} %{buildroot}%{home_dir}/ %if 0%{?rhel} >= 7 install -pm 755 %{SOURCE11} %{buildroot}%{home_dir}/.rpmmacros_rpmbuilder @@ -163,6 +165,11 @@ fi ############################################################################### %changelog +* Fri Mar 03 2017 Anton Novojilov - 1.3.0-0 +- Fixed bug with deleting directories inside rpmbuilder directory +- Added initenv utility +- buildmon refactoring + * Mon Jan 30 2017 Anton Novojilov - 1.2.2-0 - Enable password authentication by default diff --git a/rpmbuilder.spec b/rpmbuilder.spec index a48a29a..8f265e0 100644 --- a/rpmbuilder.spec +++ b/rpmbuilder.spec @@ -2,7 +2,7 @@ Summary: RPM package build helper Name: rpmbuilder -Version: 1.5.2 +Version: 1.6.0 Release: 0%{?dist} License: EKOL Group: Development/Tools @@ -51,6 +51,9 @@ rm -rf %{buildroot} ############################################################################### %changelog +* Fri Mar 03 2017 Anton Novojilov - 1.6.0-0 +- Added compatibility with latest version of rpmbuilder-node package + * Mon Jan 23 2017 Anton Novojilov - 1.5.2-0 - Improved checking remote sources availability From 774c20ee8bf315ca2972b680f2ac38468f255468 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Thu, 2 Mar 2017 18:33:42 -0500 Subject: [PATCH 2/8] Typo fix --- rpmbuilder-node.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpmbuilder-node.spec b/rpmbuilder-node.spec index 16d82bc..b87b54f 100644 --- a/rpmbuilder-node.spec +++ b/rpmbuilder-node.spec @@ -166,7 +166,7 @@ fi %changelog * Fri Mar 03 2017 Anton Novojilov - 1.3.0-0 -- Fixed bug with deleting directories inside rpmbuilder directory +- Fixed bug with deleting directories inside rpmbuild directory - Added initenv utility - buildmon refactoring From 0b7f5ff0ebec36fbf88b36558934e3bf37966175 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Thu, 2 Mar 2017 18:44:54 -0500 Subject: [PATCH 3/8] Failed validation now fail entire build --- SOURCES/rpmbuilder | 30 +++++++++++++++++++++++++----- rpmbuilder.spec | 1 + 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/SOURCES/rpmbuilder b/SOURCES/rpmbuilder index 2c2c2e5..548c948 100755 --- a/SOURCES/rpmbuilder +++ b/SOURCES/rpmbuilder @@ -81,6 +81,7 @@ ERROR_BUILD=5 ERROR_LINT=6 ERROR_REMOTE=7 ERROR_DEPS=8 +ERROR_VALIDATE=9 MACRO="name:Name version:Version release:Release url:URL vendor:Vendor epoch:Epoch" @@ -961,6 +962,8 @@ remoteBuildProcess() { showSeparator "PACKAGE VALIDATION" + has_errors="" + for package in $packages ; do package=$(echo "$package" | tr -dc '[:print:]') package_name=$(basename "$package") @@ -968,9 +971,15 @@ remoteBuildProcess() { [[ -n "$dest_dir" ]] && package_file="$dest_dir/$package_name" - validatePackage "$package_file" + if ! validatePackage "$package_file" ; then + has_errors=true + fi done + if [[ -z "$has_errors" ]] ; then + doExit $ERROR_VALIDATE + fi + if [[ -n "$post" ]] ; then show "Executing post-build script..." $($post) @@ -994,7 +1003,7 @@ remoteBuildProcess() { fi fi else - show "Error! Package was not be builded." $RED + show "Error! Can't build package." $RED if [[ -z "$verbose" ]] ; then show "Last $LOG_QUOTE_SIZE lines from build log:" $RED @@ -1198,7 +1207,7 @@ localBuildProcess() { stopStatusObserver show "" - show "Error! Package was not be builded." $RED + show "Error! Can't build package." $RED if [[ -z "$verbose" ]] ; then show "Last $LOG_QUOTE_SIZE lines from build log:" $RED @@ -1256,15 +1265,23 @@ localBuildProcess() { showSeparator "PACKAGE VALIDATION" + has_errors="" + for package in $packages ; do package_name=$(basename "$package") package_file="$CWD/$package_name" [[ -n "$dest_dir" ]] && package_file="$dest_dir/$package_name" - validatePackage "$package_file" + if ! validatePackage "$package_file" ; then + has_errors=true + fi done + if [[ -z "$has_errors" ]] ; then + doExit $ERROR_VALIDATE + fi + pushd "$CWD" &> /dev/null if [[ -n "$post" ]] ; then @@ -2602,7 +2619,10 @@ validatePackage() { if [[ -z "$has_errors" ]] ; then show "OK" $GREEN + return 0 fi + + return 1 } ############################################################################### @@ -2631,7 +2651,7 @@ doExit() { if [[ -z "$1" || "$1" == "0" ]] ; then sleep $TMUX_OK_SLEEP else - [[ -z "$keep_log" ]] && sleep $TMUX_ERROR_SLEEP + [[ -z "$keep_log" || $1 -eq $ERROR_VALIDATE ]] && sleep $TMUX_ERROR_SLEEP fi fi diff --git a/rpmbuilder.spec b/rpmbuilder.spec index 8f265e0..87ada85 100644 --- a/rpmbuilder.spec +++ b/rpmbuilder.spec @@ -53,6 +53,7 @@ rm -rf %{buildroot} %changelog * Fri Mar 03 2017 Anton Novojilov - 1.6.0-0 - Added compatibility with latest version of rpmbuilder-node package +- Failed validation now fail entire build * Mon Jan 23 2017 Anton Novojilov - 1.5.2-0 - Improved checking remote sources availability From 3c6c85714a196957ac810ebc9624a84dabf85391 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Thu, 2 Mar 2017 18:48:34 -0500 Subject: [PATCH 4/8] TMUX_ERROR_SLEEP increased to 30 min --- SOURCES/rpmbuilder | 2 +- rpmbuilder.spec | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/SOURCES/rpmbuilder b/SOURCES/rpmbuilder index 548c948..a68aa76 100755 --- a/SOURCES/rpmbuilder +++ b/SOURCES/rpmbuilder @@ -120,7 +120,7 @@ KEEP_ARGS=true SUPPORTED_ARCH_TYPES="tbz2 tar.bz2 tgz tar.gz tbz tar.bz txz tar.xz" TMUX_OK_SLEEP=5 -TMUX_ERROR_SLEEP=300 +TMUX_ERROR_SLEEP=1800 LINT_WAIT=3 diff --git a/rpmbuilder.spec b/rpmbuilder.spec index 87ada85..7acf068 100644 --- a/rpmbuilder.spec +++ b/rpmbuilder.spec @@ -54,6 +54,7 @@ rm -rf %{buildroot} * Fri Mar 03 2017 Anton Novojilov - 1.6.0-0 - Added compatibility with latest version of rpmbuilder-node package - Failed validation now fail entire build +- Minor improvements * Mon Jan 23 2017 Anton Novojilov - 1.5.2-0 - Improved checking remote sources availability From 34194d31a1cd6bc59b11958e5fdca05a6165c3cb Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Thu, 2 Mar 2017 19:07:05 -0500 Subject: [PATCH 5/8] Improved buildmon --- SOURCES/buildmon | 29 +++++++++++++++++++---------- SOURCES/rpmbuilder | 4 ++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/SOURCES/buildmon b/SOURCES/buildmon index 6e80ee4..654933d 100755 --- a/SOURCES/buildmon +++ b/SOURCES/buildmon @@ -11,7 +11,7 @@ BUILDER_USER="builder" BUILDER_DIR="/home/builder" EXEC="$BUILDER_DIR/buildmon" BUILD_LOCK="$BUILDER_DIR/.buildlock" -CLEAN_DELAY=600 +CLEAN_PERIOD=600 # 10 Min MAX_KEEP_DAYS=7 # 1 Week DIR_LIST="BUILD BUILDROOT RPMS SOURCES SPECS SRPMS" @@ -28,31 +28,40 @@ main() { } check() { - local has_lock local counter=0 while : ; do - sleep 1 + sleep 3 - if [[ $(pgrep -U $BUILDER_USER rpmbuild) ]] ; then - touch $BUILD_LOCK - has_lock=true + if isBuildInProgress ; then + touch "$BUILD_LOCK" else - if [[ -n $has_lock ]] ; then - rm -f $BUILD_LOCK - has_lock="" + if [[ -n "$has_lock" ]] ; then + rm -f "$BUILD_LOCK" fi fi ((counter++)) - if [[ $counter -eq $CLEAN_DELAY ]] ; then + if [[ $counter -gt $CLEAN_PERIOD ]] ; then counter=0 clean fi done } +isBuildInProgress() { + if [[ $(pgrep -U "$BUILDER_USER" "rpmbuild") ]] ; then + return 0 + fi + + if [[ $(pgrep -U "$BUILDER_USER" "yum-builddep") ]] ; then + return 0 + fi + + return 1 +} + clean() { for dir in $DIR_LIST ; do find "$BUILDER_DIR/rpmbuild/$dir" -maxdepth 1 -mtime +$MAX_KEEP_DAYS -delete diff --git a/SOURCES/rpmbuilder b/SOURCES/rpmbuilder index a68aa76..afe44c7 100755 --- a/SOURCES/rpmbuilder +++ b/SOURCES/rpmbuilder @@ -137,8 +137,8 @@ LOCK_TIME_MAX_DIFF=10800 # 3 Hours SSH_BASE_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=quiet" -CHECK_QUEUE_DELAY=10 CHECK_LOCAL_QUEUE_DELAY=2 +CHECK_REMOTE_QUEUE_DELAY=15 LOG_QUOTE_SIZE=50 REPO_LOG_QUOTE_SIZE=25 @@ -1037,7 +1037,7 @@ checkRemoteAvailability() { showt "\r${CL_GREEN}Executing:${CL_NORM} ${CL_BROWN}waiting in queue${CL_NORM}" while : ; do - sleep $CHECK_QUEUE_DELAY + sleep $CHECK_REMOTE_QUEUE_DELAY showt "\r${CL_GREEN}Executing:${CL_NORM} ${CL_CYAN}checking queue${CL_NORM}" From 2416013f7742981814dfc88a739d6d52fd85e0fc Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 3 Mar 2017 07:18:51 -0500 Subject: [PATCH 6/8] RPMB-123 RPMB-128 Improved golang sources packing --- SOURCES/rpmbuilder | 98 +++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/SOURCES/rpmbuilder b/SOURCES/rpmbuilder index afe44c7..4fa7cd9 100755 --- a/SOURCES/rpmbuilder +++ b/SOURCES/rpmbuilder @@ -220,8 +220,6 @@ main() { [[ -n "$bitbucket" ]] && processBitbucketSource [[ -n "$launchpad" ]] && processLaunchpadSource - [[ -n "$gopack" ]] && processGolangSource "$spec" - if [[ -n "$download" ]] ; then downloadRemoteSources "$spec" else @@ -602,36 +600,6 @@ processLaunchpadSource() { fi } -processGolangSource() { - local spec="$1" - local args sources output status - - sources=($(getSpecValue "$MACRO_SOURCE" "$spec")) - - if [[ -n "$source_dir" ]] ; then - output="${source_dir}/${sources[0]}" - else - output="${sources[0]}" - fi - - [[ -e $output ]] && return - - showSeparator "GOPACK" - - args="-o $output" - - [[ -n "$revision" ]] && args="$args -r $revision" - [[ -n "$branch" ]] && args="$args -b $branch" - [[ -n "$tag" ]] && args="$args -t $tag" - [[ -n "$verbose" ]] && args="$args -V" - - args="$args $gopack" - - gopack $args - - [[ $? -ne 0 ]] && doExit 1 -} - parseSourceURL() { local spec="$1" @@ -667,17 +635,18 @@ downloadRemoteSources() { dlcache="." fi + showSeparator "DOWNLOADING" + if [[ ! -d "$dlcache" ]] ; then show "Can't download sources - download destination directory is not exist." $RED doExit $ERROR_ARGS fi - showSeparator "DOWNLOADING" - - [[ -n "$git" ]] && packSourcesFromGit "$spec" - [[ -n "$svn" ]] && packSourcesFromSvn "$spec" - [[ -n "$bzr" ]] && packSourcesFromBzr "$spec" - [[ -n "$hg" ]] && packSourcesFromHg "$spec" + [[ -n "$git" ]] && packSourcesFromGit "$spec" + [[ -n "$svn" ]] && packSourcesFromSvn "$spec" + [[ -n "$bzr" ]] && packSourcesFromBzr "$spec" + [[ -n "$hg" ]] && packSourcesFromHg "$spec" + [[ -n "$gopack" ]] && packGolangSources "$spec" local source_file sources_list source_file_name patches_list files @@ -695,7 +664,7 @@ downloadRemoteSources() { continue fi - show "Downloading source file ${CL_CYAN}$source_file${CL_NORM}..." + show "Downloading source file ${CL_CYAN}${source_file}${CL_NORM}..." downloadSource "$source_file" "$download" @@ -725,10 +694,11 @@ remoteBuild() { $($pre) fi - [[ -n "$git" ]] && packSourcesFromGit "$spec" - [[ -n "$svn" ]] && packSourcesFromSvn "$spec" - [[ -n "$bzr" ]] && packSourcesFromBzr "$spec" - [[ -n "$hg" ]] && packSourcesFromHg "$spec" + [[ -n "$git" ]] && packSourcesFromGit "$spec" + [[ -n "$svn" ]] && packSourcesFromSvn "$spec" + [[ -n "$bzr" ]] && packSourcesFromBzr "$spec" + [[ -n "$hg" ]] && packSourcesFromHg "$spec" + [[ -n "$gopack" ]] && packGolangSources "$spec" [[ -n "$repo_spec" ]] && spec="$repo_spec" @@ -1069,10 +1039,11 @@ localBuild() { $($pre) fi - [[ -n "$git" ]] && packSourcesFromGit "$spec" - [[ -n "$svn" ]] && packSourcesFromSvn "$spec" - [[ -n "$bzr" ]] && packSourcesFromBzr "$spec" - [[ -n "$hg" ]] && packSourcesFromHg "$spec" + [[ -n "$git" ]] && packSourcesFromGit "$spec" + [[ -n "$svn" ]] && packSourcesFromSvn "$spec" + [[ -n "$bzr" ]] && packSourcesFromBzr "$spec" + [[ -n "$hg" ]] && packSourcesFromHg "$spec" + [[ -n "$gopack" ]] && packGolangSources "$spec" [[ -n "$repo_spec" ]] && spec="$repo_spec" @@ -2007,6 +1978,37 @@ packSourcesFromSvn() { show "Source files from ${CL_BOLD}svn${CL_NORM} repo packed as ${CL_CYAN}${source_name}.${source_ext}${CL_NORM}" } +packGolangSources() { + local spec="$1" + local args sources output status + + sources=($(getSpecValue "$MACRO_SOURCE" "$spec")) + + if [[ -n "$source_dir" ]] ; then + output="${source_dir}/${sources[0]}" + else + output="${sources[0]}" + fi + + if [[ -e $output ]] ; then + show "File ${CL_CYAN}${sources[0]}${CL_NORM} already created." + return + fi + + args="-o $output" + + [[ -n "$revision" ]] && args="$args -r $revision" + [[ -n "$branch" ]] && args="$args -b $branch" + [[ -n "$tag" ]] && args="$args -t $tag" + [[ -n "$verbose" ]] && args="$args -V" + + args="$args $gopack" + + gopack $args + + [[ $? -ne 0 ]] && doExit 1 +} + processSpec() { local spec="$1" @@ -2292,7 +2294,7 @@ downloadSource() { if [[ -z "$tmuxl" && -z "$dlcache" && $time_diff -gt $LONG_DOWNLOADING ]] ; then show "" show "HINT: Downloading took more than $LONG_DOWNLOADING seconds. We strongly recommend use download" $BROWN - show "cache feature in this case. Use -ds or --dlcache ." $BROWN + show "cache feature in this case. Use -dc or --dlcache ." $BROWN show "" fi fi From dec888c2f3ae7515115ca88a6c31eaa7cf1d0a49 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 3 Mar 2017 16:51:58 -0500 Subject: [PATCH 7/8] Updated CodeClimate config --- .codeclimate.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.codeclimate.yml b/.codeclimate.yml index b5e293b..8ba2d4d 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -43,3 +43,4 @@ ratings: - SOURCES/rpmbuilder - SOURCES/rpmunbuilder - SOURCES/buildmon + - SOURCES/initenv From 6b9e1280019060ba53c708e90c71411c40b02493 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 3 Mar 2017 16:58:11 -0500 Subject: [PATCH 8/8] Spec updated --- rpmbuilder.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/rpmbuilder.spec b/rpmbuilder.spec index 7acf068..c17d699 100644 --- a/rpmbuilder.spec +++ b/rpmbuilder.spec @@ -54,6 +54,7 @@ rm -rf %{buildroot} * Fri Mar 03 2017 Anton Novojilov - 1.6.0-0 - Added compatibility with latest version of rpmbuilder-node package - Failed validation now fail entire build +- Improved gloang sources packing - Minor improvements * Mon Jan 23 2017 Anton Novojilov - 1.5.2-0