Skip to content

Commit

Permalink
Issue #7546 was fixed.
Browse files Browse the repository at this point in the history
  • Loading branch information
andrii0lomakin authored and tglman committed Jul 28, 2017
1 parent 30543fc commit 9123b26
Show file tree
Hide file tree
Showing 3 changed files with 193 additions and 0 deletions.
4 changes: 4 additions & 0 deletions core/pom.xml
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@
<exclude.test.9>**/SBTreeWALTest.java</exclude.test.9>
<exclude.test.10>**/LocalPaginatedClusterWithWALTest.java</exclude.test.10>
<exclude.test.11>**/OSBTreeBonsaiWALTest.java</exclude.test.11>
<exclude.test.12>**/InvalidRemovedFileIdsTest.java</exclude.test.12>
</properties>
</profile>

Expand All @@ -105,6 +106,7 @@
<exclude.test.9>empty.java</exclude.test.9>
<exclude.test.10>empty.java</exclude.test.10>
<exclude.test.11>empty.java</exclude.test.11>
<exclude.test.12>empty.java</exclude.test.12>
</properties>

</profile>
Expand All @@ -128,6 +130,7 @@
<exclude.test.9>empty.java</exclude.test.9>
<exclude.test.10>empty.java</exclude.test.10>
<exclude.test.11>empty.java</exclude.test.11>
<exclude.test.12>empty.java</exclude.test.12>
</properties>

</profile>
Expand Down Expand Up @@ -201,6 +204,7 @@
<exclude>${exclude.test.9}</exclude>
<exclude>${exclude.test.10}</exclude>
<exclude>${exclude.test.11}</exclude>
<exclude>${exclude.test.12}</exclude>
</excludes>
<properties>
<property>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import com.orientechnologies.common.serialization.types.OIntegerSerializer;
import com.orientechnologies.common.serialization.types.OLongSerializer;
import com.orientechnologies.common.types.OModifiableBoolean;
import com.orientechnologies.common.util.OPair;
import com.orientechnologies.orient.core.command.OCommandOutputListener;
import com.orientechnologies.orient.core.config.OGlobalConfiguration;
import com.orientechnologies.orient.core.exception.OStorageException;
Expand Down Expand Up @@ -1267,6 +1268,12 @@ private OFileClassic createFileInstance(String fileName) throws InterruptedExcep
}

private void readNameIdMap() throws IOException, InterruptedException {
//older versions of ODB incorrectly logged file deletions
//some deleted files have the same id
//because we reuse ids of removed files when we re-create them
//we need to fix this situation
final Map<Integer, Set<String>> filesWithfNegativeIds = new HashMap<Integer, Set<String>>();

nameIdMap = new ConcurrentHashMap<String, Integer>();
long localFileCounter = -1;

Expand All @@ -1279,6 +1286,32 @@ private void readNameIdMap() throws IOException, InterruptedException {
if (localFileCounter < absFileId)
localFileCounter = absFileId;

final Integer existingId = nameIdMap.get(nameFileIdEntry.name);

if (existingId != null && existingId < 0) {
final Set<String> files = filesWithfNegativeIds.get(existingId);

if (files != null) {
files.remove(nameFileIdEntry.name);

if (files.isEmpty()) {
filesWithfNegativeIds.remove(existingId);
}
}
}

if (nameFileIdEntry.fileId < 0) {
Set<String> files = filesWithfNegativeIds.get(nameFileIdEntry.fileId);

if (files == null) {
files = new HashSet<String>();
files.add(nameFileIdEntry.name);
filesWithfNegativeIds.put(nameFileIdEntry.fileId, files);
} else {
files.add(nameFileIdEntry.name);
}
}

nameIdMap.put(nameFileIdEntry.name, nameFileIdEntry.fileId);
}

Expand All @@ -1305,6 +1338,18 @@ private void readNameIdMap() throws IOException, InterruptedException {
}
}
}

for (Map.Entry<Integer, Set<String>> entry : filesWithfNegativeIds.entrySet()) {
final Set<String> files = entry.getValue();

if (files.size() > 1) {
for (String fileName : files) {
fileCounter++;
final int nextId = -fileCounter;
nameIdMap.put(fileName, nextId);
}
}
}
}

private NameFileIdEntry readNextNameIdEntry() throws IOException {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
package com.orientechnologies.orient.core.storage.impl.local.paginated;

import com.orientechnologies.common.serialization.types.OIntegerSerializer;
import com.orientechnologies.common.serialization.types.OLongSerializer;
import com.orientechnologies.common.serialization.types.OStringSerializer;
import com.orientechnologies.orient.core.db.ODatabase;
import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.metadata.schema.OSchema;
import com.orientechnologies.orient.core.storage.OStorage;
import com.orientechnologies.orient.core.storage.cache.OWriteCache;
import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage;
import com.sun.xml.internal.ws.policy.AssertionSet;
import org.testng.Assert;
import org.testng.annotations.Test;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

@Test
public class InvalidRemovedFileIdsTest {

public void testRemovedFileIds() throws Exception {
final String buildDirectory = System.getProperty("buildDirectory", ".");
final String dbPath = buildDirectory + File.separator + InvalidRemovedFileIdsTest.class.getSimpleName();

deleteDirectory(new File(dbPath));

ODatabaseDocumentTx db = new ODatabaseDocumentTx("plocal:" + dbPath);
db.create();

OStorage storage = db.getStorage();
db.close();
storage.close(true, false);

final RandomAccessFile cacheState = new RandomAccessFile(new File(dbPath, "name_id_map.cm"), "rw");
cacheState.seek(cacheState.length());

writeNameIdEntry(cacheState, "c1.cpm", -100);
writeNameIdEntry(cacheState, "c1.pcl", -100);

writeNameIdEntry(cacheState, "c2.cpm", -200);
writeNameIdEntry(cacheState, "c2.pcl", -200);
writeNameIdEntry(cacheState, "c2.pcl", -400);

writeNameIdEntry(cacheState, "c3.cpm", -500);
writeNameIdEntry(cacheState, "c3.pcl", -500);
writeNameIdEntry(cacheState, "c4.cpm", -500);
writeNameIdEntry(cacheState, "c4.pcl", -600);
writeNameIdEntry(cacheState, "c4.cpm", -600);

db = new ODatabaseDocumentTx("plocal:" + dbPath);
db.open("admin", "admin");

db.set(ODatabase.ATTRIBUTES.MINIMUMCLUSTERS, 1);

final OSchema schema = db.getMetadata().getSchema();
schema.createClass("c1");
schema.createClass("c2");
schema.createClass("c3");
schema.createClass("c4");

storage = db.getStorage();
OWriteCache writeCache = ((OAbstractPaginatedStorage) storage).getWriteCache();

final Map<String, Long> files = writeCache.files();
final Set<Long> ids = new HashSet<Long>();

final Long c1_cpm_id = files.get("c1.cpm");
Assert.assertNotNull(c1_cpm_id);
Assert.assertTrue(c1_cpm_id > 0);
Assert.assertTrue(ids.add(c1_cpm_id));

final Long c1_pcl_id = files.get("c1.pcl");
Assert.assertNotNull(c1_pcl_id);
Assert.assertTrue(c1_pcl_id > 0);
Assert.assertTrue(ids.add(c1_pcl_id));

final Long c2_cpm_id = files.get("c2.cpm");
Assert.assertNotNull(c2_cpm_id);
Assert.assertTrue(ids.add(c2_cpm_id));
Assert.assertEquals(writeCache.internalFileId(c2_cpm_id), 200); //check that updated file map has been read

final Long c2_pcl_id = files.get("c2.pcl");
Assert.assertNotNull(c2_pcl_id);
Assert.assertTrue(ids.add(c2_pcl_id));
Assert.assertEquals(writeCache.internalFileId(c2_pcl_id), 400); //check that updated file map has been read

final Long c3_cpm_id = files.get("c3.cpm");
Assert.assertNotNull(c3_cpm_id);
Assert.assertTrue(c3_cpm_id > 0);
Assert.assertTrue(ids.add(c3_cpm_id));

final Long c3_pcl_id = files.get("c3.pcl");
Assert.assertNotNull(c3_pcl_id);
Assert.assertTrue(c3_pcl_id > 0);
Assert.assertTrue(ids.add(c3_pcl_id));

final Long c4_cpm_id = files.get("c4.cpm");
Assert.assertNotNull(c4_cpm_id);
Assert.assertTrue(c4_cpm_id > 0);
Assert.assertTrue(ids.add(c4_cpm_id));

final Long c4_pcl_id = files.get("c4.pcl");
Assert.assertNotNull(c4_pcl_id);
Assert.assertTrue(c1_pcl_id > 0);
Assert.assertTrue(ids.add(c4_pcl_id));

db.close();
}

private static void writeNameIdEntry(RandomAccessFile file, String name, int fileId) throws IOException {
final int nameSize = OStringSerializer.INSTANCE.getObjectSize(name);

byte[] serializedRecord = new byte[OIntegerSerializer.INT_SIZE + nameSize + OLongSerializer.LONG_SIZE];
OIntegerSerializer.INSTANCE.serializeLiteral(nameSize, serializedRecord, 0);
OStringSerializer.INSTANCE.serialize(name, serializedRecord, OIntegerSerializer.INT_SIZE);
OLongSerializer.INSTANCE.serializeLiteral(fileId, serializedRecord, OIntegerSerializer.INT_SIZE + nameSize);

file.write(serializedRecord);
}

private static void deleteDirectory(final File directory) {
if (directory.exists()) {
final File[] files = directory.listFiles();

if (files != null) {
for (File file : files) {
if (file.isDirectory()) {
deleteDirectory(file);
} else {
Assert.assertTrue(file.delete());
}
}

Assert.assertTrue(directory.delete());
}

}
}
}

0 comments on commit 9123b26

Please sign in to comment.