Merge pull request #7568 from orientechnologies/issue_7546_v2_2

Issue #7546 was fixed.

core/pom.xml

@@ -82,6 +82,7 @@
                 <exclude.test.9>**/SBTreeWALTest.java</exclude.test.9>
                 <exclude.test.10>**/LocalPaginatedClusterWithWALTest.java</exclude.test.10>
                 <exclude.test.11>**/OSBTreeBonsaiWALTest.java</exclude.test.11>
+                <exclude.test.12>**/InvalidRemovedFileIdsTest.java</exclude.test.12>
             </properties>
         </profile>
 
@@ -105,6 +106,7 @@
                 <exclude.test.9>empty.java</exclude.test.9>
                 <exclude.test.10>empty.java</exclude.test.10>
                 <exclude.test.11>empty.java</exclude.test.11>
+                <exclude.test.12>empty.java</exclude.test.12>
             </properties>
 
         </profile>
@@ -128,6 +130,7 @@
                 <exclude.test.9>empty.java</exclude.test.9>
                 <exclude.test.10>empty.java</exclude.test.10>
                 <exclude.test.11>empty.java</exclude.test.11>
+                <exclude.test.12>empty.java</exclude.test.12>
             </properties>
 
         </profile>
@@ -201,6 +204,7 @@
                         <exclude>${exclude.test.9}</exclude>
                         <exclude>${exclude.test.10}</exclude>
                         <exclude>${exclude.test.11}</exclude>
+                        <exclude>${exclude.test.12}</exclude>
                     </excludes>
                     <properties>
                         <property>

core/src/main/java/com/orientechnologies/orient/core/storage/cache/local/OWOWCache.java

@@ -1267,6 +1267,12 @@ private OFileClassic createFileInstance(String fileName) throws InterruptedExcep
   }
 
   private void readNameIdMap() throws IOException, InterruptedException {
+    //older versions of ODB incorrectly logged file deletions
+    //some deleted files have the same id
+    //because we reuse ids of removed files when we re-create them
+    //we need to fix this situation
+    final Map<Integer, Set<String>> filesWithfNegativeIds = new HashMap<Integer, Set<String>>();
+
     nameIdMap = new ConcurrentHashMap<String, Integer>();
     long localFileCounter = -1;
 
@@ -1279,6 +1285,32 @@ private void readNameIdMap() throws IOException, InterruptedException {
       if (localFileCounter < absFileId)
         localFileCounter = absFileId;
 
+      final Integer existingId = nameIdMap.get(nameFileIdEntry.name);
+
+      if (existingId != null && existingId < 0) {
+        final Set<String> files = filesWithfNegativeIds.get(existingId);
+
+        if (files != null) {
+          files.remove(nameFileIdEntry.name);
+
+          if (files.isEmpty()) {
+            filesWithfNegativeIds.remove(existingId);
+          }
+        }
+      }
+
+      if (nameFileIdEntry.fileId < 0) {
+        Set<String> files = filesWithfNegativeIds.get(nameFileIdEntry.fileId);
+
+        if (files == null) {
+          files = new HashSet<String>();
+          files.add(nameFileIdEntry.name);
+          filesWithfNegativeIds.put(nameFileIdEntry.fileId, files);
+        } else {
+          files.add(nameFileIdEntry.name);
+        }
+      }
+
       nameIdMap.put(nameFileIdEntry.name, nameFileIdEntry.fileId);
     }
 
@@ -1305,6 +1337,25 @@ private void readNameIdMap() throws IOException, InterruptedException {
         }
       }
     }
+
+    final Set<String> fixedFiles = new HashSet<String>();
+
+    for (Map.Entry<Integer, Set<String>> entry : filesWithfNegativeIds.entrySet()) {
+      final Set<String> files = entry.getValue();
+
+      if (files.size() > 1) {
+        for (String fileName : files) {
+          fileCounter++;
+          final int nextId = -fileCounter;
+          nameIdMap.put(fileName, nextId);
+
+          fixedFiles.add(fileName);
+        }
+      }
+    }
+
+    if (!fixedFiles.isEmpty())
+      OLogManager.instance().warn(this, "Removed files " + fixedFiles + " had duplicated ids. Problem is fixed automatically.");
   }
 
   private NameFileIdEntry readNextNameIdEntry() throws IOException {

core/src/test/java/com/orientechnologies/orient/core/storage/impl/local/paginated/InvalidRemovedFileIdsTest.java

@@ -0,0 +1,143 @@
+package com.orientechnologies.orient.core.storage.impl.local.paginated;
+
+import com.orientechnologies.common.serialization.types.OIntegerSerializer;
+import com.orientechnologies.common.serialization.types.OLongSerializer;
+import com.orientechnologies.common.serialization.types.OStringSerializer;
+import com.orientechnologies.orient.core.db.ODatabase;
+import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
+import com.orientechnologies.orient.core.metadata.schema.OSchema;
+import com.orientechnologies.orient.core.storage.OStorage;
+import com.orientechnologies.orient.core.storage.cache.OWriteCache;
+import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+@Test
+public class InvalidRemovedFileIdsTest {
+
+  public void testRemovedFileIds() throws Exception {
+    final String buildDirectory = System.getProperty("buildDirectory", ".");
+    final String dbPath = buildDirectory + File.separator + InvalidRemovedFileIdsTest.class.getSimpleName();
+
+    deleteDirectory(new File(dbPath));
+
+    ODatabaseDocumentTx db = new ODatabaseDocumentTx("plocal:" + dbPath);
+    db.create();
+
+    OStorage storage = db.getStorage();
+    db.close();
+    storage.close(true, false);
+
+    final RandomAccessFile fileMap = new RandomAccessFile(new File(dbPath, "name_id_map.cm"), "rw");
+    fileMap.seek(fileMap.length());
+
+    writeNameIdEntry(fileMap, "c1.cpm", -100);
+    writeNameIdEntry(fileMap, "c1.pcl", -100);
+
+    writeNameIdEntry(fileMap, "c2.cpm", -200);
+    writeNameIdEntry(fileMap, "c2.pcl", -200);
+    writeNameIdEntry(fileMap, "c2.pcl", -400);
+
+    writeNameIdEntry(fileMap, "c3.cpm", -500);
+    writeNameIdEntry(fileMap, "c3.pcl", -500);
+    writeNameIdEntry(fileMap, "c4.cpm", -500);
+    writeNameIdEntry(fileMap, "c4.pcl", -600);
+    writeNameIdEntry(fileMap, "c4.cpm", -600);
+
+    db = new ODatabaseDocumentTx("plocal:" + dbPath);
+    db.open("admin", "admin");
+
+    db.set(ODatabase.ATTRIBUTES.MINIMUMCLUSTERS, 1);
+
+    final OSchema schema = db.getMetadata().getSchema();
+    schema.createClass("c1");
+    schema.createClass("c2");
+    schema.createClass("c3");
+    schema.createClass("c4");
+
+    storage = db.getStorage();
+    OWriteCache writeCache = ((OAbstractPaginatedStorage) storage).getWriteCache();
+
+    final Map<String, Long> files = writeCache.files();
+    final Set<Long> ids = new HashSet<Long>();
+
+    final Long c1_cpm_id = files.get("c1.cpm");
+    Assert.assertNotNull(c1_cpm_id);
+    Assert.assertTrue(c1_cpm_id > 0);
+    Assert.assertTrue(ids.add(c1_cpm_id));
+
+    final Long c1_pcl_id = files.get("c1.pcl");
+    Assert.assertNotNull(c1_pcl_id);
+    Assert.assertTrue(c1_pcl_id > 0);
+    Assert.assertTrue(ids.add(c1_pcl_id));
+
+    final Long c2_cpm_id = files.get("c2.cpm");
+    Assert.assertNotNull(c2_cpm_id);
+    Assert.assertTrue(ids.add(c2_cpm_id));
+    Assert.assertEquals(writeCache.internalFileId(c2_cpm_id), 200); //check that updated file map has been read
+
+    final Long c2_pcl_id = files.get("c2.pcl");
+    Assert.assertNotNull(c2_pcl_id);
+    Assert.assertTrue(ids.add(c2_pcl_id));
+    Assert.assertEquals(writeCache.internalFileId(c2_pcl_id), 400); //check that updated file map has been read
+
+    final Long c3_cpm_id = files.get("c3.cpm");
+    Assert.assertNotNull(c3_cpm_id);
+    Assert.assertTrue(c3_cpm_id > 0);
+    Assert.assertTrue(ids.add(c3_cpm_id));
+
+    final Long c3_pcl_id = files.get("c3.pcl");
+    Assert.assertNotNull(c3_pcl_id);
+    Assert.assertTrue(c3_pcl_id > 0);
+    Assert.assertTrue(ids.add(c3_pcl_id));
+
+    final Long c4_cpm_id = files.get("c4.cpm");
+    Assert.assertNotNull(c4_cpm_id);
+    Assert.assertTrue(c4_cpm_id > 0);
+    Assert.assertTrue(ids.add(c4_cpm_id));
+
+    final Long c4_pcl_id = files.get("c4.pcl");
+    Assert.assertNotNull(c4_pcl_id);
+    Assert.assertTrue(c1_pcl_id > 0);
+    Assert.assertTrue(ids.add(c4_pcl_id));
+
+    db.close();
+  }
+
+  private static void writeNameIdEntry(RandomAccessFile file, String name, int fileId) throws IOException {
+    final int nameSize = OStringSerializer.INSTANCE.getObjectSize(name);
+
+    byte[] serializedRecord = new byte[OIntegerSerializer.INT_SIZE + nameSize + OLongSerializer.LONG_SIZE];
+    OIntegerSerializer.INSTANCE.serializeLiteral(nameSize, serializedRecord, 0);
+    OStringSerializer.INSTANCE.serialize(name, serializedRecord, OIntegerSerializer.INT_SIZE);
+    OLongSerializer.INSTANCE.serializeLiteral(fileId, serializedRecord, OIntegerSerializer.INT_SIZE + nameSize);
+
+    file.write(serializedRecord);
+  }
+
+  private static void deleteDirectory(final File directory) {
+    if (directory.exists()) {
+      final File[] files = directory.listFiles();
+
+      if (files != null) {
+        for (File file : files) {
+          if (file.isDirectory()) {
+            deleteDirectory(file);
+          } else {
+            Assert.assertTrue(file.delete());
+          }
+        }
+
+        Assert.assertTrue(directory.delete());
+      }
+
+    }
+  }
+}