manmagic3 · Oct 22, 2024 · Oct 22, 2024 · Oct 23, 2024 · Oct 24, 2024 · Oct 24, 2024
Showing 1,304 changed files with 70,659 additions and 34,232 deletions.
diff --git a/.build/README.md b/.build/README.md
@@ -71,54 +71,64 @@ Running Tests
 
 Running unit tests with docker:
 
-    .build/docker/run-tests.sh test
+    .build/docker/run-tests.sh -a test
 
 
 Running unittests without docker:
 
-    .build/run-tests.sh test
+    .build/run-tests.sh -a test
 
 
 Running only a split of unittests, with docker:
 
-    .build/docker/run-tests.sh test 1/64
+    .build/docker/run-tests.sh -a test -c 1/64
 
 
 Running unittests with a specific jdk with docker:
 
-    .build/docker/run-tests.sh test 1/64 11
+    .build/docker/run-tests.sh -a test -c 1/64 -j 11
 
 
 Running only unit tests matching a regexp, with docker:
 
-    .build/docker/run-tests.sh test VerifyTest 11
-    .build/docker/run-tests.sh test "Compaction*Test$" 11
+    .build/docker/run-tests.sh -a test -t VerifyTest -j 11
+    .build/docker/run-tests.sh -a test -t "Compaction*Test$" -j 11
 
 
 Running other types of tests with docker:
 
-    .build/docker/run-tests.sh test
-    .build/docker/run-tests.sh stress-test
-    .build/docker/run-tests.sh fqltool-test
-    .build/docker/run-tests.sh microbench
-    .build/docker/run-tests.sh test-cdc
-    .build/docker/run-tests.sh test-compression
-    .build/docker/run-tests.sh test-oa
-    .build/docker/run-tests.sh test-system-keyspace-directory
-    .build/docker/run-tests.sh test-latest
-    .build/docker/run-tests.sh test-burn
-    .build/docker/run-tests.sh long-test
-    .build/docker/run-tests.sh cqlsh-test
-    .build/docker/run-tests.sh jvm-dtest
-    .build/docker/run-tests.sh jvm-dtest-upgrade
-    .build/docker/run-tests.sh dtest
-    .build/docker/run-tests.sh dtest-novnode
-    .build/docker/run-tests.sh dtest-latest
-    .build/docker/run-tests.sh dtest-large
-    .build/docker/run-tests.sh dtest-large-novnode
-    .build/docker/run-tests.sh dtest-upgrade
-    .build/docker/run-tests.sh dtest-upgrade-large
-
+    .build/docker/run-tests.sh -a test
+    .build/docker/run-tests.sh -a stress-test
+    .build/docker/run-tests.sh -a fqltool-test
+    .build/docker/run-tests.sh -a microbench
+    .build/docker/run-tests.sh -a test-cdc
+    .build/docker/run-tests.sh -a test-compression
+    .build/docker/run-tests.sh -a test-oa
+    .build/docker/run-tests.sh -a test-system-keyspace-directory
+    .build/docker/run-tests.sh -a test-tries
+    .build/docker/run-tests.sh -a test-burn
+    .build/docker/run-tests.sh -a long-test
+    .build/docker/run-tests.sh -a cqlsh-test
+    .build/docker/run-tests.sh -a jvm-dtest
+    .build/docker/run-tests.sh -a jvm-dtest-upgrade
+    .build/docker/run-tests.sh -a dtest
+    .build/docker/run-tests.sh -a dtest-novnode
+    .build/docker/run-tests.sh -a dtest-offheap
+    .build/docker/run-tests.sh -a dtest-large
+    .build/docker/run-tests.sh -a dtest-large-novnode
+    .build/docker/run-tests.sh -a dtest-upgrade
+    .build/docker/run-tests.sh -a dtest-upgrade-large
+
+Repeating tests
+
+Just add the '-repeat' suffix to the test type and pass in the repeat arguments
+
+    .build/run-tests.sh -a jvm-dtest-repeat -t BooleanTest -e REPEATED_TESTS_COUNT=2
+    .build/docker/run-tests.sh -a jvm-dtest-repeat -t BooleanTest -e REPEATED_TESTS_COUNT=2 -j 11
+
+Fail fast with repeating tests is done with REPEATED_TESTS_STOP_ON_FAILURE
+
+    .build/run-tests.sh -a jvm-dtest-repeat -t BooleanTest -e REPEATED_TESTS_COUNT=2 -e REPEATED_TESTS_STOP_ON_FAILURE=false
 
 Running python dtests without docker:
 
@@ -134,6 +144,10 @@ Other python dtest types without docker:
 
     .build/run-python-dtests.sh dtest-upgrade-large
 
+Debugging test scripts:
+
+    DEBUG=true .build/docker/run-tests.sh -a test
+
 
 Running Sonar analysis (experimental)
 -------------------------------------

diff --git a/.build/checkstyle.xml b/.build/checkstyle.xml
@@ -62,7 +62,13 @@
       <property name="idFormat" value="blockPathToFile"/>
       <property name="influenceFormat" value="0"/>
     </module>
-
+
+    <module name="SuppressWithNearbyCommentFilter">
+      <property name="commentFormat" value="checkstyle: permit this invocation"/>
+      <property name="idFormat" value="blockToCases"/>
+      <property name="influenceFormat" value="0"/>
+    </module>
+
     <module name="RegexpSinglelineJava">
       <!-- block system time -->
       <property name="id" value="blockSystemClock"/>
@@ -151,6 +157,14 @@
       <property name="message" value="Avoid Short() and use Short.valueOf()" />
     </module>
 
+    <module name="RegexpSinglelineJava">
+      <!-- block toLowerCase and toUpperCase -->
+      <property name="id" value="blockToCases"/>
+      <property name="format" value="toLowerCase\(|toUpperCase\("/>
+      <property name="ignoreComments" value="true"/>
+      <property name="message" value="Avoid toLowerCase() or toUpperCase() and use LocalizeString class instead" />
+    </module>
+
     <module name="SuppressionCommentFilter">
       <property name="offCommentFormat" value="checkstyle: suppress below '([\w\|]+)'"/>
       <property name="idFormat" value="$1"/>
@@ -169,6 +183,10 @@
       <property name="message" value="Use the CassandraRelevantProperties or CassandraRelevantEnv instead." />
     </module>
 
+    <module name="IllegalType"> <!-- usage of var check -->
+      <property name="illegalClassNames" value="var"/>
+    </module>
+
     <module name="RedundantImport"/>
     <module name="UnusedImports"/>
 

diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh
@@ -15,27 +15,70 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#
 # A wrapper script to run-tests.sh (or dtest-python.sh) in docker.
-#  Can split (or grep) the test list into multiple docker runs, collecting results.
+#
 
 [ $DEBUG ] && set -x
 
-# help
-if [ "$#" -lt 1 ] || [ "$#" -gt 3 ] || [ "$1" == "-h" ]; then
-    echo ""
-    echo "Usage: run-tests.sh test_type [split_chunk|test_regexp] [java_version]"
-    echo ""
-    echo "        default split_chunk is 1/1"
-    echo "        default java_version is what 'java.default' specifies in build.xml"
-    exit 1
-fi
+print_help() {
+  echo ""
+  echo "Usage: $0 [-a|-t|-c|-j|-h] [extra arguments]"
+  echo "   -a Test target type: test, test-compression, test-cdc, ..."
+  echo "   -t Test name regexp to run."
+  echo "   -c Chunk to run in the form X/Y: Run chunk X from a total of Y chunks."
+  echo "   -j Java version. Default java_version is what 'java.default' specifies in build.xml."
+  echo "   [extra arguments] will be passed to downstream scripts."
+  exit 1
+}
 
 error() {
-    echo >&2 $2;
-    set -x
-    exit $1
+  echo >&2 $2;
+  set -x
+  exit $1
 }
 
+# legacy argument handling
+case ${1} in
+  "build_dtest_jars" | "stress-test" | "fqltool-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
+    test_type="-a ${1}"
+    if [[ -z ${2} ]]; then
+      test_list=""
+    elif [[ -n ${2} && "${2}" =~ ^[0-9]+/[0-9]+$ ]]; then
+      test_list="-c ${2}";
+    else
+      test_list="-t ${2}";
+    fi
+    if [[ -n ${3} ]]; then java_version="-j ${3}"; else java_version=""; fi
+    echo "Using deprecated legacy arguments.  Please update to new parameter format: ${test_type} ${test_list} ${java_version}"
+    $0 ${test_type} ${test_list} ${java_version}
+    exit $?
+esac
+
+env_vars=""
+while getopts ":a:t:c:e:hj:" opt; do
+  # shellcheck disable=SC2220
+  # Invalid flags check disabled as we'll pass them to other scripts
+  case $opt in
+    a ) test_target="$OPTARG"
+        ;;
+    t ) test_name_regexp="$OPTARG"
+        ;;
+    c ) chunk="$OPTARG"
+        ;;
+    j ) java_version="$OPTARG"
+        ;;
+    e ) env_vars="${env_vars} -e $OPTARG"
+        ;;
+    h ) print_help
+        exit 0
+        ;;
+    e)  ;; # Repeat vars are just passed to downstream run-tests-enhaced.sh
+    \?) die "Invalid option: -$OPTARG"
+        ;;
+  esac
+done
+
 # variables, with defaults
 [ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname "$0")/../..)"
 [ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest"
@@ -53,10 +96,11 @@ command -v timeout >/dev/null 2>&1 || { error 1 "timeout needs to be installed";
 [ -f "${cassandra_dir}/.build/run-tests.sh" ] || { error 1 "${cassandra_dir}/.build/run-tests.sh must exist"; }
 
 # arguments
-target=$1
+target=${test_target}
 split_chunk="1/1"
-[ "$#" -gt 1 ] && split_chunk=$2
-java_version=$3
+split_chunk=${chunk-'1/1'}
+test_name_regexp=${test_name_regexp}
+java_version=${java_version}
 
 test_script="run-tests.sh"
 java_version_default=`grep 'property\s*name="java.default"' ${cassandra_dir}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'`
@@ -135,7 +179,7 @@ esac
 
 # figure out resource limits, scripts, and mounts for the test type
 docker_flags="-m 5g --memory-swap 5g"
-case ${target} in
+case ${test_target/-repeat/} in
     "build_dtest_jars")
     ;;
     "stress-test" | "fqltool-test" )
@@ -156,7 +200,7 @@ case ${target} in
         [[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
         docker_flags="-m 15g --memory-swap 15g"
     ;;
-    "test"| "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
+    "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
         [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
     ;;
     *)
@@ -236,8 +280,10 @@ logfile="${build_dir}/test/logs/docker_attach_${container_name}.log"
 # Docker commands:
 #  set java to java_version
 #  execute the run_script
+[ -n "${test_name_regexp}" ] && test_name_regexp_arg="-t ${test_name_regexp}" || split_chunk_arg="-c ${split_chunk}"
+
 docker_command="source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_version} ; \
-            \${CASSANDRA_DIR}/.build/docker/_docker_init_tests.sh ${target} ${split_chunk} ; exit \$?"
+            \${CASSANDRA_DIR}/.build/docker/_docker_init_tests.sh -a ${target} ${split_chunk_arg} ${test_name_regexp_arg} ${env_vars} ; exit \$?"
 
 # start the container, timeout after 4 hours
 docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_envs} ${docker_mounts} ${docker_volume_opt} ${image_name} sleep 4h)

diff --git a/.build/owasp/dependency-check-suppressions.xml b/.build/owasp/dependency-check-suppressions.xml
@@ -33,14 +33,25 @@
     </suppress>
 
     <!-- https://issues.apache.org/jira/browse/CASSANDRA-19142 -->
+    <!-- https://issues.apache.org/jira/browse/CASSANDRA-20412 -->
     <suppress>
         <packageUrl regex="true">^pkg:maven/ch\.qos\.logback/logback\-core@.*$</packageUrl>
         <cve>CVE-2023-6378</cve>
         <cve>CVE-2023-6481</cve>
+        <cve>CVE-2024-12798</cve>
+        <cve>CVE-2024-12801</cve>
     </suppress>
     <suppress>
         <packageUrl regex="true">^pkg:maven/ch\.qos\.logback/logback\-classic@.*$</packageUrl>
         <cve>CVE-2023-6378</cve>
         <cve>CVE-2023-6481</cve>
+        <cve>CVE-2024-12798</cve>
+        <cve>CVE-2024-12801</cve>
+    </suppress>
+
+    <!-- https://issues.apache.org/jira/browse/CASSANDRA-20024 -->
+    <suppress>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.lucene/lucene\-.*@9.7.0$</packageUrl>
+        <cve>CVE-2024-45772</cve>
     </suppress>
 </suppressions>
diff --git a/.build/parent-pom-template.xml b/.build/parent-pom-template.xml
@@ -302,7 +302,7 @@
       <dependency>
         <groupId>com.github.luben</groupId>
         <artifactId>zstd-jni</artifactId>
-        <version>1.5.5-1</version>
+        <version>1.5.7-2</version>
       </dependency>
       <dependency>
         <groupId>com.google.guava</groupId>
@@ -1219,12 +1219,12 @@
       <dependency>
         <groupId>org.apache.lucene</groupId>
         <artifactId>lucene-core</artifactId>
-        <version>9.7.0</version>
+        <version>9.12.0</version>
       </dependency>
       <dependency>
         <groupId>org.apache.lucene</groupId>
         <artifactId>lucene-analysis-common</artifactId>
-        <version>9.7.0</version>
+        <version>9.12.0</version>
       </dependency>
       <dependency>
         <groupId>io.github.jbellis</groupId>

diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh
@@ -28,18 +28,41 @@
 [ $DEBUG ] && set -x
 
 # help
-if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ "$1" == "-h" ]; then
+if [ "$#" -lt 1 ] || [ "$1" == "-h" ]; then
     echo ""
-    echo "Usage: run-python-dtest.sh test_type [split_chunk|test_regexp]"
+    echo "Usage: $0 [-a|-t|-c|-j|-h]"
+    echo "   -a Test target type: dtest, dtest-latest, ..."
+    echo "   -t Test name regexp to run."
+    echo "   -c Chunk to run in the form X/Y: Run chunk X from a total of Y chunks."
     echo ""
     echo "        default split_chunk is 1/1"
     exit 1
 fi
 
 # Pass in target to run, defaults to dtest
-DTEST_TARGET="${1:-dtest}"
-# Optional: pass in chunk to test, formatted as "K/N" for the Kth chunk of N chunks
-DTEST_SPLIT_CHUNK="$2"
+DTEST_TARGET="dtest"
+
+# TODO implement repeated runs, eg CASSANDRA-18942
+while getopts "a:t:c:hj:" opt; do
+  case $opt in
+    a ) DTEST_TARGET="$OPTARG"
+        ;;
+    t ) DTEST_SPLIT_CHUNK="$OPTARG"
+        ;;
+    c ) DTEST_SPLIT_CHUNK="$OPTARG"
+        ;;
+    h ) print_help
+        exit 0
+        ;;
+    j ) ;; # To avoid failing on java_version param from docker/run_tests.sh
+    \?) error 1 "Invalid option: -$OPTARG"
+        ;;
+  esac
+done
+shift $((OPTIND-1))
+if [ "$#" -ne 0 ]; then
+  error 1 "Unexpected arguments"
+fi
 
 # variables, with defaults
 [ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)"