Patch 3

.github/workflows/sonarcloud.yml

@@ -21,7 +21,6 @@ on:
   push:
     branches:
       - master
-    types: [closed]
 
 jobs:
   build:
@@ -79,6 +78,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
+          if [ ! "$SONAR_TOKEN" ]; then
+            echo "SONAR_TOKEN environment is null, skip check"
+          exit 0
+          fi
           cd fe
           mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_incubator-doris -DskipTests
 # scanner be

.licenserc.yaml

@@ -37,7 +37,6 @@ header:
     - "**/*.sql"
     - "**/*.lock"
     - "**/*.out"
-    - "tsan_suppressions"
     - "docs/.markdownlintignore"
     - "fe/fe-core/src/test/resources/data/net_snmp_normal"
     - "fe/fe-core/src/main/antlr4/org/apache/doris/nereids/JavaLexer.g4"
@@ -81,6 +80,11 @@ header:
     - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_parquet.hql"
     - "docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/"
     - "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl"
+    - "conf/mysql_ssl_default_certificate/*"
+    - "conf/mysql_ssl_default_certificate/client_certificate/ca.pem"
+    - "conf/mysql_ssl_default_certificate/client_certificate/client-cert.pem"
+    - "conf/mysql_ssl_default_certificate/client_certificate/client-key.pem"
+    - "regression-test/ssl_default_certificate/*"
     - "extension/beats/go.mod"
     - "extension/beats/go.sum"
 

.rat-excludes

@@ -35,5 +35,4 @@ condition_variable.*
 murmur_hash3.*
 utf8_check.cpp
 .markdownlintignore
-tsan_suppressions
 sse2neon.h

README.md

@@ -37,7 +37,7 @@ Apache Doris is an easy-to-use, high-performance and real-time analytical databa
 All this makes Apache Doris an ideal tool for scenarios including report analysis, ad-hoc query, unified data warehouse, and data lake query acceleration. On Apache Doris, users can build various applications, such as user behavior analysis, AB test platform, log retrieval analysis, user portrait analysis, and order analysis.
 
 
-🎉 Version 1.2.3 released now! It is fully evolved release and all users are encouraged to upgrade to this release. Check out the 🔗[Release Notes](https://doris.apache.org/docs/dev/releasenotes/release-1.2.3) here. 
+🎉 Version 1.2.4 released now! It is fully evolved release and all users are encouraged to upgrade to this release. Check out the 🔗[Release Notes](https://doris.apache.org/docs/dev/releasenotes/release-1.2.4) here. 
 
 🎉 Version 1.1.5 released now. It is a stability improvement and bugfix release based on version 1.1. Check out the 🔗[Release Notes](https://doris.apache.org/docs/dev/releasenotes/release-1.1.5) here. 
 

be/CMakeLists.txt

@@ -1016,6 +1016,7 @@ install(FILES
     ${BASE_DIR}/../conf/be.conf
     ${BASE_DIR}/../conf/odbcinst.ini
     ${BASE_DIR}/../conf/asan_suppr.conf
+    ${BASE_DIR}/../conf/lsan_suppr.conf
     ${BASE_DIR}/../conf/hdfs-site.xml
     DESTINATION ${OUTPUT_DIR}/conf)
 

be/src/common/exception.h

@@ -45,10 +45,7 @@ class Exception : public std::exception {
     Exception(int code, const std::string_view fmt, Args&&... args)
             : Exception(code, fmt::format(fmt, std::forward<Args>(args)...)) {}
 
-    std::string code_as_string() const {
-        return (int)_code >= 0 ? doris::to_string(static_cast<TStatusCode::type>(_code))
-                               : fmt::format("E{}", (int16_t)_code);
-    }
+    std::string code_as_string() const { return fmt::format("E{}", _code); }
 
     int code() const { return _code; }
 

be/src/common/expected.h

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#ifndef __cplusplus
+#error need cpp support
+#endif
+
+#if __cplusplus <= 202002L
+#include "util/expected.hpp"
+namespace doris {
+template <typename T, typename E>
+using expected = tl::expected<T, E>;
+template <typename E>
+using unexpected = tl::unexpected<E>;
+template <class E>
+using bad_expected_access = tl::bad_expected_access<E>;
+using unexpect_t = tl::unexpect_t;
+using tl::unexpect; // NOLINT
+} // namespace doris
+#else
+#include <expected>
+namespace doris {
+template <typename T, typename E>
+using expected = std::expected<T, E>;
+template <typename E>
+using unexpected = std::unexpected<E>;
+template <class E>
+using bad_expected_access = std::bad_expected_access<E>;
+using unexpect_t = std::unexpect_t;
+using std::unexpect; // NOLINT
+} // namespace doris
+#endif

be/src/common/factory_creator.h

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+#include <memory>
+
+// All object should inherit from this class, like
+// class A  {
+// DISALLOW_EXPILICT_NEW(A);
+// };
+//
+// Then the caller could not call new A() any more, has to call A::create_shared(...)
+// or A::create_unique(...), then we could make sure all object in our project is shared
+// pointer.
+// But could call A a(...);
+
+// A macro to disallow the copy constructor and operator= functions
+// This should be used in the private: declarations for a class
+//
+// Not use template like cowhelper to implements this feature because it has problem
+// during inherits
+// TODO try to allow make_unique
+//
+#define ENABLE_FACTORY_CREATOR(TypeName)                                                     \
+private:                                                                                     \
+    void* operator new(std::size_t size) { return ::operator new(size); }                    \
+    void* operator new[](std::size_t size) { return ::operator new[](size); }                \
+                                                                                             \
+public:                                                                                      \
+    void* operator new(std::size_t count, void* ptr) { return ::operator new(count, ptr); }  \
+    void operator delete(void* ptr) noexcept { ::operator delete(ptr); }                     \
+    void operator delete[](void* ptr) noexcept { ::operator delete[](ptr); }                 \
+    void operator delete(void* ptr, void* place) noexcept { ::operator delete(ptr, place); } \
+    template <typename... Args>                                                              \
+    static std::shared_ptr<TypeName> create_shared(Args&&... args) {                         \
+        return std::make_shared<TypeName>(std::forward<Args>(args)...);                      \
+    }                                                                                        \
+    template <typename... Args>                                                              \
+    static std::unique_ptr<TypeName> create_unique(Args&&... args) {                         \
+        return std::unique_ptr<TypeName>(new TypeName(std::forward<Args>(args)...));         \
+    }

be/src/exprs/block_bloom_filter.hpp

@@ -78,6 +78,13 @@ class BlockBloomFilter {
         }
     }
 
+    // This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    void insert_crc32_hash(const Slice& key) noexcept {
+        if (key.data) {
+            insert(HashUtil::crc_hash(key.data, key.size, _hash_seed));
+        }
+    }
+
 #ifdef __AVX2__
 
     static inline ATTRIBUTE_ALWAYS_INLINE __attribute__((__target__("avx2"))) __m256i make_mark(
@@ -124,6 +131,14 @@ class BlockBloomFilter {
         }
     }
 
+    // This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    bool find_crc32_hash(const Slice& key) const noexcept {
+        if (key.data) {
+            return find(HashUtil::crc_hash(key.data, key.size, _hash_seed));
+        } else {
+            return false;
+        }
+    }
     // Computes the logical OR of this filter with 'other' and stores the result in this
     // filter.
     // Notes:

be/src/exprs/bloom_filter_func.h

@@ -53,8 +53,23 @@ class BloomFilterAdaptor {
         return _bloom_filter->find(data);
     }
 
+    // This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    template <typename T>
+    bool test_new_hash(T data) const {
+        if constexpr (std::is_same_v<T, Slice>) {
+            return _bloom_filter->find_crc32_hash(data);
+        } else {
+            return _bloom_filter->find(data);
+        }
+    }
+
     void add_bytes(const char* data, size_t len) { _bloom_filter->insert(Slice(data, len)); }
 
+    // This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    void add_bytes_new_hash(const char* data, size_t len) {
+        _bloom_filter->insert_crc32_hash(Slice(data, len));
+    }
+
     // test_element/find_element only used on vectorized engine
     template <typename T>
     bool test_element(T element) const {
@@ -198,8 +213,14 @@ class BloomFilterFuncBase {
 
     virtual void insert(const void* data) = 0;
 
+    // This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    virtual void insert_crc32_hash(const void* data) = 0;
+
     virtual bool find(const void* data) const = 0;
 
+    // This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    virtual bool find_crc32_hash(const void* data) const = 0;
+
     virtual bool find_olap_engine(const void* data) const = 0;
 
     virtual bool find_uint32_t(uint32_t data) const = 0;
@@ -343,13 +364,32 @@ struct StringFindOp {
             bloom_filter.add_bytes(value->data, value->size);
         }
     }
+
+    // This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    void insert_crc32_hash(BloomFilterAdaptor& bloom_filter, const void* data) const {
+        const auto* value = reinterpret_cast<const StringRef*>(data);
+        if (value) {
+            bloom_filter.add_bytes_new_hash(value->data, value->size);
+        }
+    }
+
     bool find(const BloomFilterAdaptor& bloom_filter, const void* data) const {
         const auto* value = reinterpret_cast<const StringRef*>(data);
         if (value == nullptr) {
             return false;
         }
         return bloom_filter.test(Slice(value->data, value->size));
     }
+
+    //This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    bool find_crc32_hash(const BloomFilterAdaptor& bloom_filter, const void* data) const {
+        const auto* value = reinterpret_cast<const StringRef*>(data);
+        if (value == nullptr) {
+            return false;
+        }
+        return bloom_filter.test_new_hash(Slice(value->data, value->size));
+    }
+
     bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const {
         return StringFindOp::find(bloom_filter, data);
     }
@@ -463,6 +503,18 @@ class BloomFilterFunc final : public BloomFilterFuncBase {
         dummy.insert(*_bloom_filter, data);
     }
 
+    // This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    void insert_crc32_hash(const void* data) override {
+        if constexpr (std::is_same_v<typename BloomFilterTypeTraits<type>::FindOp, StringFindOp> ||
+                      std::is_same_v<typename BloomFilterTypeTraits<type>::FindOp,
+                                     FixedStringFindOp>) {
+            DCHECK(_bloom_filter != nullptr);
+            dummy.insert_crc32_hash(*_bloom_filter, data);
+        } else {
+            insert(data);
+        }
+    }
+
     void insert_fixed_len(const char* data, const int* offsets, int number) override {
         DCHECK(_bloom_filter != nullptr);
         dummy.insert_batch(*_bloom_filter, data, offsets, number);
@@ -489,6 +541,17 @@ class BloomFilterFunc final : public BloomFilterFuncBase {
         return dummy.find(*_bloom_filter, data);
     }
 
+    // This function is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    bool find_crc32_hash(const void* data) const override {
+        if constexpr (std::is_same_v<typename BloomFilterTypeTraits<type>::FindOp, StringFindOp> ||
+                      std::is_same_v<typename BloomFilterTypeTraits<type>::FindOp,
+                                     FixedStringFindOp>) {
+            DCHECK(_bloom_filter != nullptr);
+            return dummy.find_crc32_hash(*_bloom_filter, data);
+        }
+        return find(data);
+    }
+
     bool find_olap_engine(const void* data) const override {
         return dummy.find_olap_engine(*_bloom_filter, data);
     }

be/src/exprs/runtime_filter.cpp

@@ -353,7 +353,8 @@ class RuntimePredicateWrapper {
               _fragment_instance_id(params->fragment_instance_id),
               _filter_id(params->filter_id),
               _use_batch(IRuntimeFilter::enable_use_batch(_state->be_exec_version(),
-                                                          _column_return_type)) {}
+                                                          _column_return_type)),
+              _use_new_hash(_state->be_exec_version() >= 2) {}
     // for a 'tmp' runtime predicate wrapper
     // only could called assign method or as a param for merge
     RuntimePredicateWrapper(RuntimeState* state, ObjectPool* pool, PrimitiveType column_type,
@@ -366,7 +367,8 @@ class RuntimePredicateWrapper {
               _fragment_instance_id(fragment_instance_id),
               _filter_id(filter_id),
               _use_batch(IRuntimeFilter::enable_use_batch(_state->be_exec_version(),
-                                                          _column_return_type)) {}
+                                                          _column_return_type)),
+              _use_new_hash(_state->be_exec_version() >= 2) {}
     // init runtime filter wrapper
     // alloc memory to init runtime filter function
     Status init(const RuntimeFilterParams* params) {
@@ -430,7 +432,12 @@ class RuntimePredicateWrapper {
                 }
             } else {
                 while (it->has_next()) {
-                    bloom_filter->insert(it->get_value());
+                    if (_use_new_hash) {
+                        bloom_filter->insert_crc32_hash(it->get_value());
+                    } else {
+                        bloom_filter->insert(it->get_value());
+                    }
+
                     it->next();
                 }
             }
@@ -453,12 +460,20 @@ class RuntimePredicateWrapper {
             break;
         }
         case RuntimeFilterType::BLOOM_FILTER: {
-            _context.bloom_filter_func->insert(data);
+            if (_use_new_hash) {
+                _context.bloom_filter_func->insert_crc32_hash(data);
+            } else {
+                _context.bloom_filter_func->insert(data);
+            }
             break;
         }
         case RuntimeFilterType::IN_OR_BLOOM_FILTER: {
             if (_is_bloomfilter) {
-                _context.bloom_filter_func->insert(data);
+                if (_use_new_hash) {
+                    _context.bloom_filter_func->insert_crc32_hash(data);
+                } else {
+                    _context.bloom_filter_func->insert(data);
+                }
             } else {
                 _context.hybrid_set->insert(data);
             }
@@ -1036,6 +1051,10 @@ class RuntimePredicateWrapper {
 
     // When _column_return_type is invalid, _use_batch will be always false.
     bool _use_batch;
+
+    // When _use_new_hash is set to true, use the new hash method.
+    // This is only to be used if the be_exec_version may be less than 2. If updated, please delete it.
+    const bool _use_new_hash;
 };
 
 Status IRuntimeFilter::create(RuntimeState* state, ObjectPool* pool, const TRuntimeFilterDesc* desc,