Merge branch 'main' into bitset_block

dev-tools/scripts/releaseWizard.py

@@ -49,6 +49,7 @@
 from collections import OrderedDict
 from datetime import datetime
 from datetime import timedelta
+from datetime import timezone
 
 try:
     import holidays
@@ -99,7 +100,7 @@ def expand_jinja(text, vars=None):
         'state': state,
         'gpg_key' : state.get_gpg_key(),
         'gradle_cmd' : 'gradlew.bat' if is_windows() else './gradlew',
-        'epoch': unix_time_millis(datetime.utcnow()),
+        'epoch': unix_time_millis(datetime.now(tz=timezone.utc)),
         'get_next_version': state.get_next_version(),
         'current_git_rev': state.get_current_git_rev(),
         'keys_downloaded': keys_downloaded(),
@@ -199,7 +200,7 @@ def check_prerequisites(todo=None):
     return True
 
 
-epoch = datetime.utcfromtimestamp(0)
+epoch = datetime.fromtimestamp(timestamp=0, tz=timezone.utc)
 
 
 def unix_time_millis(dt):
@@ -279,7 +280,7 @@ def __init__(self, config_path, release_version, script_version):
         self.latest_version = None
         self.previous_rcs = {}
         self.rc_number = 1
-        self.start_date = unix_time_millis(datetime.utcnow())
+        self.start_date = unix_time_millis(datetime.now(tz=timezone.utc))
         self.script_branch = run("git rev-parse --abbrev-ref HEAD").strip()
         self.mirrored_versions = None
         try:
@@ -741,7 +742,7 @@ def get_vars(self):
 
     def set_done(self, is_done):
         if is_done:
-            self.state['done_date'] = unix_time_millis(datetime.utcnow())
+            self.state['done_date'] = unix_time_millis(datetime.now(tz=timezone.utc))
             if self.persist_vars:
                 for k in self.persist_vars:
                     self.state[k] = self.get_vars()[k]
@@ -935,7 +936,7 @@ def expand_multiline(cmd_txt, indent=0):
 
 
 def unix_to_datetime(unix_stamp):
-    return datetime.utcfromtimestamp(unix_stamp / 1000)
+    return datetime.fromtimestamp(timestamp=unix_stamp / 1000, tz=timezone.utc)
 
 
 def generate_asciidoc():
@@ -949,7 +950,7 @@ def generate_asciidoc():
 
     fh.write("= Lucene Release %s\n\n" % state.release_version)
     fh.write("(_Generated by releaseWizard.py v%s at %s_)\n\n"
-             % (getScriptVersion(), datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC")))
+             % (getScriptVersion(), datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC")))
     fh.write(":numbered:\n\n")
     fh.write("%s\n\n" % template('help'))
     for group in state.todo_groups:
@@ -1839,17 +1840,17 @@ def create_ical(todo): # pylint: disable=unused-argument
     return True
 
 
-today = datetime.utcnow().date()
+today = datetime.now(tz=timezone.utc).date()
 sundays = {(today + timedelta(days=x)): 'Sunday' for x in range(10) if (today + timedelta(days=x)).weekday() == 6}
-y = datetime.utcnow().year
+y = datetime.now(tz=timezone.utc).year
 years = [y, y+1]
 non_working = holidays.CA(years=years) + holidays.US(years=years) + holidays.UK(years=years) \
               + holidays.DE(years=years) + holidays.NO(years=years) + holidays.IN(years=years) + holidays.RU(years=years)
 
 
 def vote_close_72h_date():
     # Voting open at least 72 hours according to ASF policy
-    return datetime.utcnow() + timedelta(hours=73)
+    return datetime.now(tz=timezone.utc) + timedelta(hours=73)
 
 
 def vote_close_72h_holidays():

lucene/CHANGES.txt

@@ -11,7 +11,7 @@ API Changes
 
 New Features
 ---------------------
-(No changes)
+* GITHUB#14097: Binary partitioning merge policy over float-valued vector field. (Mike Sokolov)
 
 Improvements
 ---------------------
@@ -41,6 +41,9 @@ API Changes
 * GITHUB#14069: Added DocIdSetIterator#intoBitSet API to let implementations
   optimize loading doc IDs into a bit set. (Adrien Grand)
 
+* GITHUB#14134: Added Bits#applyMask API to help apply live docs as a mask on a
+  bit set of matches. (Adrien Grand)
+
 New Features
 ---------------------
 (No changes)
@@ -50,6 +53,9 @@ Improvements
 
 * GITHUB#14079: Hunspell Dictionary now supports an option to tolerate REP rule count mismatches.
   (Robert Muir)
+* GITHUB#13984: Add HNSW graph checks and stats to CheckIndex
+
+* GITHUB#14113: Remove unnecessary ByteArrayDataInput allocations from `Lucene90DocValuesProducer$TermsDict.decompressBlock`. (Ankit Jain)
 
 Optimizations
 ---------------------
@@ -61,11 +67,20 @@ Optimizations
 
 Bug Fixes
 ---------------------
-(No changes)
+
+* GITHUB#14109: prefetch may select the wrong memory segment for
+  multi-segment slices. (Chris Hegarty)
+
+* GITHUB#14123: SortingCodecReader NPE when segment has no (points, vectors, etc...) (Mike Sokolov)
 
 Other
 ---------------------
-(No changes)
+
+* GITHUB#14081: Fix urls describing why NIOFS is not recommended for Windows (Marcel Yeonghyeon Ko)
+
+* GITHUB#14116 Use CDL to block threads to avoid flaky tests. (Ao Li)
+
+* GITHUB#14091: Cover all DataType. (Lu Xugang)
 
 ======================= Lucene 10.1.0 =======================
 

lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java

@@ -488,6 +488,11 @@ public int entryNode() {
       throw new UnsupportedOperationException();
     }
 
+    @Override
+    public int maxConn() {
+      throw new UnsupportedOperationException();
+    }
+
     @Override
     public NodesIterator getNodesOnLevel(int level) {
       throw new UnsupportedOperationException();

lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90OnHeapHnswGraph.java

@@ -198,6 +198,11 @@ public int entryNode() {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public int maxConn() {
+    return maxConn;
+  }
+
   @Override
   public NodesIterator getNodesOnLevel(int level) {
     throw new UnsupportedOperationException();

lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java

@@ -548,6 +548,11 @@ public int entryNode() {
       return entryNode;
     }
 
+    @Override
+    public int maxConn() {
+      return (int) bytesForConns / Integer.BYTES - 1;
+    }
+
     @Override
     public NodesIterator getNodesOnLevel(int level) {
       if (level == 0) {

lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91OnHeapHnswGraph.java

@@ -160,6 +160,11 @@ public int entryNode() {
     return entryNode;
   }
 
+  @Override
+  public int maxConn() {
+    return maxConn;
+  }
+
   @Override
   public NodesIterator getNodesOnLevel(int level) {
     if (level == 0) {

lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java

@@ -459,6 +459,11 @@ public int entryNode() {
       return entryNode;
     }
 
+    @Override
+    public int maxConn() {
+      return (int) bytesForConns / Integer.BYTES - 1;
+    }
+
     @Override
     public NodesIterator getNodesOnLevel(int level) {
       if (level == 0) {

lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java

@@ -448,6 +448,7 @@ private static final class OffHeapHnswGraph extends HnswGraph {
     final int size;
     final long bytesForConns;
     final long bytesForConns0;
+    final int maxConn;
 
     int arcCount;
     int arcUpTo;
@@ -463,6 +464,7 @@ private static final class OffHeapHnswGraph extends HnswGraph {
       this.bytesForConns = Math.multiplyExact(Math.addExact(entry.M, 1L), Integer.BYTES);
       this.bytesForConns0 =
           Math.multiplyExact(Math.addExact(Math.multiplyExact(entry.M, 2L), 1), Integer.BYTES);
+      maxConn = entry.M;
     }
 
     @Override
@@ -501,6 +503,11 @@ public int numLevels() {
       return numLevels;
     }
 
+    @Override
+    public int maxConn() {
+      return maxConn;
+    }
+
     @Override
     public int entryNode() {
       return entryNode;

lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java

@@ -537,6 +537,11 @@ public int entryNode() throws IOException {
       return entryNode;
     }
 
+    @Override
+    public int maxConn() {
+      return currentNeighborsBuffer.length / 2;
+    }
+
     @Override
     public NodesIterator getNodesOnLevel(int level) {
       if (level == 0) {

lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java

@@ -334,6 +334,11 @@ public int numLevels() {
         return graph.numLevels();
       }
 
+      @Override
+      public int maxConn() {
+        return graph.maxConn();
+      }
+
       @Override
       public int entryNode() {
         throw new UnsupportedOperationException("Not supported on a mock graph");

lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java

@@ -366,6 +366,11 @@ public int entryNode() {
         throw new UnsupportedOperationException("Not supported on a mock graph");
       }
 
+      @Override
+      public int maxConn() {
+        throw new UnsupportedOperationException("Not supported on a mock graph");
+      }
+
       @Override
       public NodesIterator getNodesOnLevel(int level) {
         if (level == 0) {

lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java

@@ -75,7 +75,7 @@ protected KnnVectorsReader() {}
    * TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO}.
    *
    * <p>The behavior is undefined if the given field doesn't have KNN vectors enabled on its {@link
-   * FieldInfo}. The return value is never {@code null}.
+   * FieldInfo}.
    *
    * @param field the vector field to search
    * @param target the vector-valued query
@@ -103,7 +103,7 @@ public abstract void search(
    * TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO}.
    *
    * <p>The behavior is undefined if the given field doesn't have KNN vectors enabled on its {@link
-   * FieldInfo}. The return value is never {@code null}.
+   * FieldInfo}.
    *
    * @param field the vector field to search
    * @param target the vector-valued query