Incremental backup does not introduce GC overhead.

orientechnologies · Aug 3, 2020 · 4943965 · 4943965
1 parent 291f7c5
commit 4943965
Show file tree

Hide file tree

Showing 150 changed files with 604 additions and 488 deletions.
diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/cache/OCacheEntry.java b/core/src/main/java/com/orientechnologies/orient/core/storage/cache/OCacheEntry.java
@@ -107,4 +107,6 @@ public interface OCacheEntry {
   void clearPageOperations();
 
   void addPageOperationRecord(PageOperationRecord pageOperationRecord);
+
+  boolean insideCache();
 }
diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/cache/OCacheEntryImpl.java b/core/src/main/java/com/orientechnologies/orient/core/storage/cache/OCacheEntryImpl.java
@@ -34,15 +34,24 @@ public final class OCacheEntryImpl implements OCacheEntry {
 
   private LRUList container;
 
-  /** Protected by page lock inside disk cache */
+  /**
+   * Protected by page lock inside disk cache
+   */
   private boolean allocatedPage;
 
-  /** Protected by page lock inside disk cache */
+  /**
+   * Protected by page lock inside disk cache
+   */
   private List<PageOperationRecord> pageOperationRecords;
 
   private int hash;
+  private final boolean insideCache;
 
-  public OCacheEntryImpl(final long fileId, final int pageIndex, final OCachePointer dataPointer) {
+  public OCacheEntryImpl(
+      final long fileId,
+      final int pageIndex,
+      final OCachePointer dataPointer,
+      final boolean insideCache) {
     if (fileId < 0) {
       throw new IllegalStateException("File id has invalid value " + fileId);
     }
@@ -55,6 +64,7 @@ public OCacheEntryImpl(final long fileId, final int pageIndex, final OCachePoint
     this.pageIndex = pageIndex;
 
     this.dataPointer = dataPointer;
+    this.insideCache = insideCache;
   }
 
   @Override
@@ -290,14 +300,25 @@ public LRUList getContainer() {
     return container;
   }
 
+  @Override
+  public boolean insideCache() {
+    return insideCache;
+  }
+
   @Override
   public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
 
     OCacheEntryImpl that = (OCacheEntryImpl) o;
 
-    if (fileId != that.fileId) return false;
+    if (fileId != that.fileId) {
+      return false;
+    }
     return pageIndex == that.pageIndex;
   }
 

diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/cache/OReadCache.java b/core/src/main/java/com/orientechnologies/orient/core/storage/cache/OReadCache.java
@@ -63,6 +63,12 @@ OCacheEntry loadForRead(
       boolean verifyChecksums)
       throws IOException;
 
+  OCacheEntry silentLoadForRead(
+      final long extFileId,
+      final int pageIndex,
+      final OWriteCache writeCache,
+      final boolean verifyChecksums);
+
   void releaseFromRead(OCacheEntry cacheEntry, OWriteCache writeCache);
 
   void releaseFromWrite(OCacheEntry cacheEntry, OWriteCache writeCache, boolean changed);

diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/cache/chm/AsyncReadCache.java b/core/src/main/java/com/orientechnologies/orient/core/storage/cache/chm/AsyncReadCache.java
@@ -40,8 +40,9 @@
  * heap memory.
  */
 public final class AsyncReadCache implements OReadCache {
-  private static final int NCPU = Runtime.getRuntime().availableProcessors();
-  private static final int WRITE_BUFFER_MAX_BATCH = 128 * ceilingPowerOfTwo(NCPU);
+
+  private static final int N_CPU = Runtime.getRuntime().availableProcessors();
+  private static final int WRITE_BUFFER_MAX_BATCH = 128 * ceilingPowerOfTwo(N_CPU);
 
   private final ConcurrentHashMap<PageKey, OCacheEntry> data;
   private final Lock evictionLock = new ReentrantLock();
@@ -126,6 +127,69 @@ public final OCacheEntry loadForRead(
     return doLoad(fileId, (int) pageIndex, writeCache, verifyChecksums);
   }
 
+  @Override
+  public final OCacheEntry silentLoadForRead(
+      final long extFileId,
+      final int pageIndex,
+      final OWriteCache writeCache,
+      final boolean verifyChecksums) {
+    final long fileId = OAbstractWriteCache.checkFileIdCompatibility(writeCache.getId(), extFileId);
+    final PageKey pageKey = new PageKey(fileId, pageIndex);
+
+    for (; ; ) {
+      OCacheEntry cacheEntry = data.get(pageKey);
+
+      if (cacheEntry != null) {
+        if (cacheEntry.acquireEntry()) {
+          return cacheEntry;
+        }
+      } else {
+        final OCacheEntry[] updatedEntry = new OCacheEntry[1];
+
+        cacheEntry =
+            data.compute(
+                pageKey,
+                (page, entry) -> {
+                  if (entry == null) {
+                    try {
+                      final OCachePointer pointer =
+                          writeCache.load(
+                              fileId, pageIndex, new OModifiableBoolean(), verifyChecksums);
+                      if (pointer == null) {
+                        return null;
+                      }
+
+                      updatedEntry[0] =
+                          new OCacheEntryImpl(
+                              page.getFileId(), page.getPageIndex(), pointer, false);
+                      return null;
+                    } catch (final IOException e) {
+                      throw OException.wrapException(
+                          new OStorageException(
+                              "Error during loading of page " + pageIndex + " for file " + fileId),
+                          e);
+                    }
+
+                  } else {
+                    return entry;
+                  }
+                });
+
+        if (cacheEntry == null) {
+          cacheEntry = updatedEntry[0];
+        }
+
+        if (cacheEntry == null) {
+          return null;
+        }
+
+        if (cacheEntry.acquireEntry()) {
+          return cacheEntry;
+        }
+      }
+    }
+  }
+
   private OCacheEntry doLoad(
       final long extFileId,
       final int pageIndex,
@@ -172,7 +236,8 @@ private OCacheEntry doLoad(
                       }
 
                       cacheSize.incrementAndGet();
-                      return new OCacheEntryImpl(page.getFileId(), page.getPageIndex(), pointer);
+                      return new OCacheEntryImpl(page.getFileId(), page.getPageIndex(), pointer,
+                          true);
                     } catch (final IOException e) {
                       throw OException.wrapException(
                           new OStorageException(
@@ -220,7 +285,8 @@ private OCacheEntry addNewPagePointerToTheCache(final long fileId, final int pag
     final OCachePointer cachePointer = new OCachePointer(pointer, bufferPool, fileId, pageIndex);
     cachePointer.incrementReadersReferrer();
 
-    final OCacheEntry cacheEntry = new OCacheEntryImpl(fileId, pageIndex, cachePointer);
+    final OCacheEntry cacheEntry = new OCacheEntryImpl(fileId, pageIndex, cachePointer,
+        true);
     cacheEntry.acquireEntry();
 
     final OCacheEntry oldCacheEntry = data.putIfAbsent(pageKey, cacheEntry);
@@ -247,6 +313,10 @@ public final void changeMaximumAmountOfMemory(final long maxMemory) {
   @Override
   public final void releaseFromRead(final OCacheEntry cacheEntry, final OWriteCache writeCache) {
     cacheEntry.releaseEntry();
+
+    if (!cacheEntry.insideCache()) {
+      cacheEntry.getCachePointer().decrementReadersReferrer();
+    }
   }
 
   @Override
@@ -255,7 +325,7 @@ public final void releaseFromWrite(
     final OCachePointer cachePointer = cacheEntry.getCachePointer();
     assert cachePointer != null;
 
-    final PageKey pageKey = new PageKey(cacheEntry.getFileId(), (int) cacheEntry.getPageIndex());
+    final PageKey pageKey = new PageKey(cacheEntry.getFileId(), cacheEntry.getPageIndex());
     if (cacheEntry.isNewlyAllocatedPage() || changed) {
       if (cacheEntry.isNewlyAllocatedPage()) {
         cacheEntry.clearAllocationFlag();

diff --git a/...nt/core/storage/impl/local/paginated/atomicoperations/OAtomicOperationBinaryTracking.java b/...nt/core/storage/impl/local/paginated/atomicoperations/OAtomicOperationBinaryTracking.java
@@ -258,7 +258,9 @@ public OCacheEntry addPage(long fileId) {
     changesContainer.maxNewPageIndex = filledUpTo;
     pageChangesContainer.delegate =
         new OCacheEntryImpl(
-            fileId, (int) filledUpTo, new OCachePointer(null, null, fileId, (int) filledUpTo));
+            fileId, (int) filledUpTo, new OCachePointer(null, null, fileId,
+            (int) filledUpTo), false);
+
     return pageChangesContainer;
   }
 

diff --git a/...ologies/orient/core/storage/impl/local/paginated/atomicoperations/OCacheEntryChanges.java b/...ologies/orient/core/storage/impl/local/paginated/atomicoperations/OCacheEntryChanges.java
@@ -188,6 +188,11 @@ public void addPageOperationRecord(PageOperationRecord pageOperationRecord) {
     delegate.addPageOperationRecord(pageOperationRecord);
   }
 
+  @Override
+  public boolean insideCache() {
+    return delegate.insideCache();
+  }
+
   @Override
   public OCacheEntry getNext() {
     throw new UnsupportedOperationException();

diff --git a/...ain/java/com/orientechnologies/orient/core/storage/memory/ODirectMemoryOnlyDiskCache.java b/...ain/java/com/orientechnologies/orient/core/storage/memory/ODirectMemoryOnlyDiskCache.java
@@ -213,6 +213,12 @@ public final OCacheEntry loadForRead(
     return cacheEntry;
   }
 
+  @Override
+  public OCacheEntry silentLoadForRead(
+      long extFileId, int pageIndex, OWriteCache writeCache, boolean verifyChecksums) {
+    return loadForRead(extFileId, pageIndex, false, writeCache, verifyChecksums);
+  }
+
   private OCacheEntry doLoad(final long fileId, final long pageIndex) {
     final int intId = extractFileId(fileId);
 
@@ -497,7 +503,9 @@ private OCacheEntry addNewPage() {
               new OCachePointer(pointer, bufferPool, id, (int) index);
           cachePointer.incrementReferrer();
 
-          cacheEntry = new OCacheEntryImpl(composeFileId(storageId, id), (int) index, cachePointer);
+          cacheEntry =
+              new OCacheEntryImpl(composeFileId(storageId, id), (int) index,
+                  cachePointer, true);
 
           final OCacheEntry oldCacheEntry = content.putIfAbsent(index, cacheEntry);
 

diff --git a/...src/test/java/com/orientechnologies/orient/core/storage/cache/chm/WTinyLFUPolicyTest.java b/...src/test/java/com/orientechnologies/orient/core/storage/cache/chm/WTinyLFUPolicyTest.java
@@ -3,6 +3,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+
 import com.orientechnologies.common.directmemory.OByteBufferPool;
 import com.orientechnologies.common.directmemory.ODirectMemoryAllocator;
 import com.orientechnologies.orient.core.storage.cache.OCacheEntry;
@@ -677,7 +678,7 @@ private static void generateEntries(
       OCacheEntry[] cacheEntries, OCachePointer[] cachePointers, OByteBufferPool pool) {
     for (int i = 0; i < cacheEntries.length; i++) {
       final OCachePointer cachePointer = new OCachePointer(pool.acquireDirect(true), pool, 1, i);
-      final OCacheEntry cacheEntry = new OCacheEntryImpl(1, i, cachePointer);
+      final OCacheEntry cacheEntry = new OCacheEntryImpl(1, i, cachePointer, false);
 
       cachePointer.incrementReadersReferrer();
       cacheEntries[i] = cacheEntry;