gh-14127: remove duplicate neighbors when writing HNSW graphs (#14157)

lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java

@@ -408,6 +408,7 @@ private int[][] writeGraph(OnHeapHnswGraph graph) throws IOException {
     // write vectors' neighbours on each level into the vectorIndex file
     int countOnLevel0 = graph.size();
     int[][] offsets = new int[graph.numLevels()][];
+    int[] scratch = new int[graph.maxConn() * 2];
     for (int level = 0; level < graph.numLevels(); level++) {
       int[] sortedNodes = NodesIterator.getSortedNodes(graph.getNodesOnLevel(level));
       offsets[level] = new int[sortedNodes.length];
@@ -417,18 +418,26 @@ private int[][] writeGraph(OnHeapHnswGraph graph) throws IOException {
         int size = neighbors.size();
         // Write size in VInt as the neighbors list is typically small
         long offsetStart = vectorIndex.getFilePointer();
-        vectorIndex.writeVInt(size);
-        // Destructively modify; it's ok we are discarding it after this
         int[] nnodes = neighbors.nodes();
         Arrays.sort(nnodes, 0, size);
         // Now that we have sorted, do delta encoding to minimize the required bits to store the
         // information
-        for (int i = size - 1; i > 0; --i) {
+        int actualSize = 0;
+        if (size > 0) {
+          scratch[0] = nnodes[0];
+          actualSize = 1;
+        }
+        for (int i = 1; i < size; i++) {
           assert nnodes[i] < countOnLevel0 : "node too large: " + nnodes[i] + ">=" + countOnLevel0;
-          nnodes[i] -= nnodes[i - 1];
+          if (nnodes[i - 1] == nnodes[i]) {
+            continue;
+          }
+          scratch[actualSize++] = nnodes[i] - nnodes[i - 1];
         }
-        for (int i = 0; i < size; i++) {
-          vectorIndex.writeVInt(nnodes[i]);
+        // Write the size after duplicates are removed
+        vectorIndex.writeVInt(actualSize);
+        for (int i = 0; i < actualSize; i++) {
+          vectorIndex.writeVInt(scratch[i]);
         }
         offsets[level][nodeOffsetId++] =
             Math.toIntExact(vectorIndex.getFilePointer() - offsetStart);

lucene/core/src/java/org/apache/lucene/util/hnsw/HnswUtil.java

@@ -129,6 +129,7 @@ static List<Component> components(
         }
         Component component =
             markRooted(hnsw, level, connectedNodes, notFullyConnected, maxConn, nextClear);
+        assert component.start() == nextClear;
         assert component.size() > 0;
         components.add(component);
         total += component.size();