MDBF-800 Extend package minor upgrade tests to ensure transition

master-libvirt/master.cfg

@@ -367,7 +367,7 @@ for builder_name in builders_install:
                 "dist_name": os_name,
                 "version_name": os_info[os_info_name]["version_name"],
                 "arch": build_arch,
-                "test_mode": "server",
+                "test_mode": "all",
                 "test_type": "minor",
                 "BB_CI": True,
                 "artifactsURL": artifactsURL,

scripts/bash_lib.sh

@@ -221,7 +221,7 @@ rpm_repoquery() {
 wait_for_mariadb_upgrade() {
   res=1
   for i in {1..20}; do
-    if pgrep -if 'mysql_upgrade|mysqlcheck|mysqlrepair|mysqlanalyze|mysqloptimize|mariadb-upgrade|mariadb-check'; then
+    if pgrep -ifa 'mysql_upgrade|mysqlcheck|mysqlrepair|mysqlanalyze|mysqloptimize|mariadb-upgrade|mariadb-check'; then
       bb_log_info "wait for mysql_upgrade to finish ($i)"
       sleep 5
     else
@@ -488,9 +488,18 @@ control_mariadb_server() {
 }
 
 store_mariadb_server_info() {
+# We need sudo here because the mariadb local root configured this way, not because we want special write permissions for the resulting file.
+# pre-commit check has an issue with it, so instead of adding an exception before each line, we add a piped sort, which should be bogus,
   sudo mariadb --skip-column-names -e "select @@version" | awk -F'-' '{ print $1 }' >"/tmp/version.$1"
-  sudo mariadb --skip-column-names -e "select engine, support, transactions, savepoints from information_schema.engines" | sort >"/tmp/engines.$1"
-  sudo mariadb --skip-column-names -e "select plugin_name, plugin_status, plugin_type, plugin_library, plugin_license from information_schema.all_plugins" | sort >"/tmp/plugins.$1"
+  sudo mariadb --skip-column-names -e "select engine, support, transactions, savepoints from information_schema.engines order by engine" | sort > "./engines.$1"
+  sudo mariadb --skip-column-names -e "select plugin_name, plugin_status, plugin_type, plugin_library, plugin_license \
+                                       from information_schema.all_plugins order by plugin_name" | sort > "./plugins.$1"
+  sudo mariadb --skip-column-names -e "select 'Stat' t, variable_name name, variable_value val
+                                       from information_schema.global_status where variable_name like '%have%' \
+                                       union \
+                                       select 'Vars' t, variable_name name, variable_value val \
+                                       from information_schema.global_variables where variable_name like '%have%' \
+                                       order by t, name" | sort > "./capabilities.$1"
 }
 
 check_upgraded_versions() {
@@ -523,9 +532,139 @@ check_upgraded_versions() {
     fi
   fi
   if diff -u /tmp/version.old /tmp/version.new; then
-    bb_log_err "server version has not changed after upgrade"
-    bb_log_err "it can be a false positive if we forgot to bump version after release,"
+    bb_log_err "Server version has not changed after upgrade."
+    bb_log_err "It can be a false positive if we forgot to bump version after release"
     bb_log_err "or if it is a development tree that is based on an old version"
     exit 1
   fi
+
+  res=0
+  errors=""
+  engines_disappeared_or_changed=$(comm -23 ./engines.old ./engines.new | wc -l)
+  set +x
+  if ((engines_disappeared_or_changed != 0)); then
+    bb_log_err "Found changes in the list of engines:"
+    diff -u ./engines.old ./engines.new
+    errors="$errors engines,"
+    res=1
+  fi
+  if [[ $test_type == "minor" ]]; then
+
+    # We are using temporary files for further comparison, because we will need to make
+    # some adjustments to them to avoid false positives, but we want original files
+    # to be stored by buildbot
+    for f in ldd-main ldd-columnstore reqs-main reqs-columnstore plugins capabilities; do
+      if [ -e "./${f}.old" ] ; then
+        cp "./${f}.old" "./${f}.old.cmp"
+        cp "./${f}.new" "./${f}.new.cmp"
+      fi
+    done
+
+    # Permanent adjustments
+    sed -i '/libstdc++.so.6(GLIBCXX/d;/libstdc++.so.6(CXXABI/d' ./reqs-*.cmp
+    # Don't compare subversions of libsystemd.so (e.g. libsystemd.so.0(LIBSYSTEMD_227)(64bit))
+    sed -i '/libsystemd.so.[0-9]*(LIBSYSTEMD_/d' ./reqs-*.cmp
+    # Ignore shells, the number of which always changes
+    sed -i '/^\/bin\/sh$/d' ./reqs-*.cmp
+
+    # Here is the place for temporary adjustments,
+    # when legitimate changes in dependencies happen between minor versions.
+    # The adjustments should be done to .cmp files, and removed after the release
+    #
+    # End of temporary adjustments
+
+    set -o pipefail
+    bb_log_info "Comparing old and new ldd output for installed binaries"
+    # We are not currently comparing it for Columnstore, too much trouble for nothing
+    if ! diff -U1000 ./ldd-main.old.cmp ./ldd-main.new.cmp | (grep -E '^[-+]|^ =' || true) ; then
+      bb_log_err "Found changes in ldd output, see above"
+      errors="$errors ldd,"
+      res=1
+    else
+      bb_log_info "ldd OK"
+    fi
+
+    bb_log_info "Comparing old and new package requirements"
+    # We are not currently comparing it for Columnstore, but we may need to in the future
+    if ! diff -U150 ./reqs-main.old.cmp ./reqs-main.new.cmp | (grep -E '^ [^ ]|^[-+]' || true) ; then
+      bb_log_err "Found changes in package requirements, see above"
+      errors="$errors requirements,"
+      res=1
+    else
+      bb_log_info "Package requirements OK"
+    fi
+
+    bb_log_info "Comparing old and new server capabilities ('%have%' variables)"
+    if ! diff -u ./capabilities.old.cmp ./capabilities.new.cmp ; then
+      bb_log_err "Found changes in server capabilities, see above"
+      errors="$errors capabilities,"
+      res=1
+    else
+      bb_log_info "Capabilities OK"
+    fi
+
+    echo "Comparing old and new plugins"
+    # Columnstore version changes often, we'll ignore it
+    grep -i columnstore ./plugins.old.cmp > /tmp/columnstore.old
+    grep -i columnstore ./plugins.new.cmp > /tmp/columnstore.new
+    if ! diff -u /tmp/columnstore.old /tmp/columnstore.new ; then
+      bb_log_warn "Columnstore version changed. Downgraded to a warning as it is usually intentional"
+      sed -i '/COLUMNSTORE/d;/Columnstore/d' ./plugins.*.cmp
+    fi
+
+    if ! diff -u ./plugins.old.cmp ./plugins.new.cmp ; then
+      bb_log_err "Found changes in available plugins, see above"
+      errors="$errors plugins,"
+      res=1
+    else
+      bb_log_info "Plugins OK"
+    fi
+  fi
+  if [ -n "$errors" ] ; then
+    bb_log_err "Problems were found with:$errors see above output for details"
+  fi
+  exit $res
+}
+
+# Collect package requirements and ldd for all binaries included into packages.
+# Expects "old" and "new" as the first argument,
+# "deb" or "rpm" as the second argument,
+# and $package_list to be set by the time of execution
+collect_dependencies() {
+  [[ $test_type == "minor" ]] || return
+  old_or_new=$1
+  pkgtype=$2
+  if rpm --noartifact ; then
+    # --noartifact option is needed to avoid build-id in rpm -lp output
+    NOARTIFACT="--noartifact"
+  fi
+  bb_log_info "Collecting dependencies for the ${old_or_new} server"
+  set +x
+  for p in ${package_list} ${spider_package_list} ; do
+    if [[ "$p" =~ columnstore ]] ; then
+      suffix="columnstore"
+    else
+      suffix="main"
+    fi
+    echo "-----------------"  >> "./reqs-${suffix}.${old_or_new}"
+    echo "$p:" >> "./reqs-${suffix}.${old_or_new}"
+    if [ "$pkgtype" == "rpm" ] ; then
+      rpm -q -R "$p" | awk '{print $1}' | grep -vE '/usr/bin/env|/usr/bin/bash' >> "./reqs-${suffix}.${old_or_new}"
+      # NOARTIFACT is either an empty string or --noartifact option, it won't be globbed or word-splitted
+      # shellcheck disable=SC2086
+      filelist=$(rpm $NOARTIFACT -ql "$p" | sort)
+    else
+      # We need sudo here for the apt-cache command, not for redirection
+      # shellcheck disable=SC2024
+      sudo apt-cache depends "$p" --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances >> "./reqs-${suffix}.${old_or_new}"
+      filelist=$(dpkg-query -L "$p" | sort)
+    fi
+    echo "====== Package $p" >> "./ldd-${suffix}.${old_or_new}"
+    for f in $filelist ; do
+      sudo ldd "$f" > /dev/null 2>&1 || continue
+      echo "=== $f" >> "./ldd-${suffix}.${old_or_new}"
+      sudo ldd "$f" | sort | sed 's/(.*)//' >> "./ldd-${suffix}.${old_or_new}"
+    done
+  done
+  set -x
 }

scripts/deb-upgrade.sh

@@ -146,42 +146,10 @@ fi
 # Check that the server is functioning and create some structures
 check_mariadb_server_and_create_structures
 
-# Store information about server version and available plugins/engines before upgrade
-if [[ $test_mode == "all" ]]; then
-  # Due to MDEV-14560, we have to restart the server to get the full list of engines
-  # MDEV-14560 is fixed in 10.2
-  if [ "${prev_major_version}" != 10.2 ]; then
-    control_mariadb_server restart
-  fi
-fi
-
+# Store information about the server before upgrade
+collect_dependencies old deb
 store_mariadb_server_info old
 
-# //TEMP todo, see below
-# # Store dependency information for old binaries/libraries:
-# # - names starting with "mysql*" in the directory where mysqld is located;
-# # - names starting with "mysql*" in the directory where mysql is located;
-# # - everything in the plugin directories installed by any MariaDB packages
-# set +x
-# for i in $(sudo which mysqld | sed -e 's/mysqld$/mysql\*/') $(which mysql | sed -e 's/mysql$/mysql\*/') $(dpkg-query -L $(dpkg -l | grep mariadb | awk '{print $2}' | xargs) | grep -v 'mysql-test' | grep -v '/debug/' | grep '/plugin/' | sed -e 's/[^\/]*$/\*/' | sort | uniq | xargs); do
-#   echo "=== $i"
-#   ldd "$i" | sort | sed 's/(.*)//'
-# done >/buildbot/ldd.old
-# set -x
-
-# # setup repository
-# sudo sh -c "echo 'deb [trusted=yes] https://deb.mariadb.org/$master_branch/$dist_name $version_name main' >/etc/apt/sources.list.d/galera-test-repo.list"
-# # Update galera-test-repo.list to point at either the galera-3 or galera-4 test repo
-# //TEMP
-# case "$branch" in
-#   *10.[1-3]*)
-#     sudo sed -i 's/repo/repo3/' /etc/apt/sources.list.d/galera-test-repo.list
-#     ;;
-#   *10.[4-9]*)
-#     sudo sed -i 's/repo/repo4/' /etc/apt/sources.list.d/galera-test-repo.list
-#     ;;
-# esac
-
 if [[ $test_mode == "deps" ]]; then
   # For the dependency check, only keep the local repo //TEMP what does this do???
   sudo sh -c "grep -iE 'deb .*file|deb-src .*file' /etc/apt/sources.list.backup >/etc/apt/sources.list"
@@ -192,10 +160,7 @@ else
 fi
 sudo rm /etc/apt/preferences.d/release
 
-# We also need official mirror for dependencies not available in BB artifacts
-# (Galera) //TEMP seems to not be necessary anymore (galera packages build in
-# artifacts?)
-# deb_setup_mariadb_mirror "$master_branch"
+deb_setup_bb_galera_artifacts_mirror
 deb_setup_bb_artifacts_mirror
 apt_get_update
 
@@ -249,8 +214,8 @@ fi
 # available
 check_mariadb_server_and_verify_structures
 
-# Store information about server version and available plugins/engines after
-# upgrade
+# Store information about the server after upgrade
+collect_dependencies new deb
 store_mariadb_server_info new
 
 # //TEMP what needs to be done here?

scripts/rpm-upgrade.sh

@@ -44,11 +44,11 @@ rpm_setup_mariadb_mirror "$prev_major_version"
 case $test_mode in
   all)
     # retrieve full package list from repo
-    pkg_list=$(rpm_repoquery) ||
+    package_list=$(rpm_repoquery) ||
       bb_log_err "unable to retrieve package list from repository"
     ;;
   server)
-    pkg_list="MariaDB-server MariaDB-client"
+    package_list="MariaDB-server MariaDB-client"
     ;;
   *)
     bb_log_err "unknown test mode ($test_mode)"
@@ -59,26 +59,26 @@ esac
 # case $test_mode in
 #   all | deps | columnstore)
 #     if [[ $test_mode == "all" ]]; then
-#       if echo "$pkg_list" | grep -qi columnstore; then
+#       if echo "$package_list" | grep -qi columnstore; then
 #         bb_log_warn "due to MCOL-4120 and other issues, Columnstore upgrade will be tested separately"
 #       fi
-#       package_list=$(echo "$pkg_list" | grep MariaDB |
+#       package_list=$(echo "$package_list" | grep MariaDB |
 #         grep -viE 'galera|columnstore' | sed -e 's/<name>//' |
 #         sed -e 's/<\/name>//' | sort -u | xargs)
 #     elif [[ $test_mode == "deps" ]]; then
-#       package_list=$(echo "$pkg_list" |
+#       package_list=$(echo "$package_list" |
 #         grep -iE 'MariaDB-server|MariaDB-test|MariaDB-client|MariaDB-common|MariaDB-compat' |
 #         sed -e 's/<name>//' | sed -e 's/<\/name>//' | sort -u | xargs)
 #     elif [[ $test_mode == "columnstore" ]]; then
-#       if ! echo "$pkg_list" | grep -q columnstore; then
+#       if ! echo "$package_list" | grep -q columnstore; then
 #         bb_log_warn "columnstore was not found in the released packages, the test will not be run"
 #         exit
 #       fi
 #       package_list="MariaDB-server MariaDB-columnstore-engine"
 #     fi
 
 #     if [[ $arch == ppc* ]]; then
-#       package_list=$(echo "$pkg_list" | xargs -n1 | sed -e 's/MariaDB-compat//gi' | xargs)
+#       package_list=$(echo "$package_list" | xargs -n1 | sed -e 's/MariaDB-compat//gi' | xargs)
 #     fi
 #     ;;
 #   server)
@@ -90,7 +90,7 @@ esac
 #     ;;
 # esac
 
-bb_log_info "Package_list: $pkg_list"
+bb_log_info "Package_list: $package_list"
 
 # # //TEMP this needs to be implemented once we have SLES VM in new BB
 # # Prepare yum/zypper configuration for installation of the last release
@@ -149,7 +149,7 @@ bb_log_info "Package_list: $pkg_list"
 sudo "$pkg_cmd" clean all
 
 # Install previous release
-echo "$pkg_list" | xargs sudo "$pkg_cmd" -y install ||
+echo "$package_list" | xargs sudo "$pkg_cmd" -y install ||
   bb_log_err "installation of a previous release failed, see the output above"
 #fi
 
@@ -179,8 +179,8 @@ fi
 
 check_mariadb_server_and_create_structures
 
-# Store information about server version and available plugins/engines before
-# upgrade
+# Store information about the server before upgrade
+collect_dependencies old rpm
 store_mariadb_server_info old
 
 # If the tested branch has the same version as the public repository,
@@ -252,10 +252,10 @@ rpm_setup_bb_galera_artifacts_mirror
 rpm_setup_bb_artifacts_mirror
 if [[ $test_type == "major" ]]; then
   # major upgrade (remove then install)
-  echo "$pkg_list" | xargs sudo "$pkg_cmd" -y install
+  echo "$package_list" | xargs sudo "$pkg_cmd" -y install
 else
   # minor upgrade (upgrade works)
-  echo "$pkg_list" | xargs sudo "$pkg_cmd" -y upgrade
+  echo "$package_list" | xargs sudo "$pkg_cmd" -y upgrade
 fi
 # set +e
 
@@ -292,12 +292,12 @@ fi
 if [[ $major_version == "$development_branch" ]] || [[ $test_type == "major" ]]; then
   sudo mariadb-upgrade
 fi
-set +e
 
 # Check that the server is functioning and previously created structures are available
 check_mariadb_server_and_verify_structures
 
-# Store information about server version and available plugins/engines after upgrade
+# Store information about the server after upgrade
+collect_dependencies new rpm
 store_mariadb_server_info new
 
 # # Dependency information for new binaries/libraries