Merge pull request #96 from upserve/linearized_tree

Completely remove cache
upserve · Jul 23, 2019 · a03663e · a03663e
2 parents 0cdc4c9 + 9471802
commit a03663e
Show file tree

Hide file tree

Showing 78 changed files with 2,458 additions and 2,662 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 build/
 .idea/
 *.iml
+logs/
diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,7 @@
 language: java
 sudo: false
 jdk:
-  - oraclejdk9
+  - oraclejdk11
 script: ./gradlew clean build
 after_success:
   - bash <(curl -s https://codecov.io/bash)
diff --git a/README.md b/README.md
@@ -53,5 +53,26 @@ To benchmark Uppend:
 
 ```sh
 ./gradlew clean fatJar
-java -jar build/libs/uppend-all-*.jar benchmark --help
+java -jar build/libs/uppend-all-*.jar --help
 ```
+
+To run tests without verbose output
+```sh
+ ./gradlew test -i
+```
+
+To run tests in a specific path
+```sh
+ ./gradlew test --tests com.upserve.uppend.blobs*
+```
+
+Example script to fork the benchmark with a system resource monitor like IOSTAT
+```sh
+trap "kill 0" EXIT
+
+java -jar build/libs/uppend-all-0.0.2-91-g6abbf45.dirty.jar benchmark ../foo/test & BENCHMARK_PID=$!
+iostat -d 2 & IOSTAT_PID=$!
+
+wait $BENCHMARK_PID
+kill $IOSTAT_PID
+```
diff --git a/build.gradle b/build.gradle
@@ -13,6 +13,8 @@ plugins {
 }
 
 import org.apache.tools.ant.filters.ExpandProperties
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat
+import org.gradle.api.tasks.testing.logging.TestLogEvent
 
 if (new File('.git').exists() && (exec {
     commandLine "sh", "-c", "git --version"
@@ -30,10 +32,6 @@ group 'com.upserve'
 
 description = """Uppend: fast, append-only key-multivalue store"""
 
-task wrapper(type: Wrapper) {
-    gradleVersion = '4.4'
-}
-
 // TODO unused-dependency is broken - claims all dependencies are unused!
 //gradleLint.rules += 'unused-dependency'
 
@@ -42,6 +40,10 @@ apply plugin: 'maven'
 apply plugin: 'jacoco'
 apply plugin: 'com.bmuschko.nexus'
 
+jacoco {
+    toolVersion = "0.8.2" // Fixed to resolve issue with JDK 11 in Gradle 4.X.Y
+}
+
 sourceCompatibility = 1.9
 targetCompatibility = 1.9
 // Requires 1.9 or greater due to unsafe memory access in MappebByteBuffer - Oracle Incident Report 9119653
@@ -52,8 +54,7 @@ repositories {
 
 dependencies {
     compile 'com.google.guava:guava:21.0'
-    compile 'com.github.ben-manes.caffeine:caffeine:2.6.2'
-    compile 'info.picocli:picocli:3.0.0'
+    compile 'info.picocli:picocli:4.0.1'
     compile 'io.dropwizard.metrics:metrics-core:3.2.3'
     compile 'it.unimi.dsi:fastutil:7.0.13'
 //    compile 'me.lemire.integercompression:JavaFastPFOR:0.1.11'
@@ -62,12 +63,6 @@ dependencies {
     testCompile 'junit:junit:4.12'
     testCompile 'org.apache.logging.log4j:log4j-api:2.8'
     testCompile 'org.mockito:mockito-core:2.18.3'
-
-    gradleLint.ignore {
-        // gradle-lint doesn't like log4j because it's not used directly
-        compile 'org.apache.logging.log4j:log4j-core:2.8'
-        compile 'org.apache.logging.log4j:log4j-slf4j-impl:2.8'
-    }
 }
 
 tasks.withType(JavaCompile) {
@@ -93,6 +88,10 @@ processResources {
 }
 
 task fatJar(type: Jar) {
+    dependencies {
+        compile 'org.apache.logging.log4j:log4j-core:2.8'
+        compile 'org.apache.logging.log4j:log4j-slf4j-impl:2.8'
+    }
     manifest {
         attributes 'Main-Class': 'com.upserve.uppend.Uppend'
     }
@@ -101,6 +100,44 @@ task fatJar(type: Jar) {
     with jar
 }
 
+tasks.withType(Test) {
+    // From https://stackoverflow.com/a/36130467/2136991
+    testLogging {
+        // set options for log level LIFECYCLE
+        events TestLogEvent.FAILED,
+                TestLogEvent.PASSED,
+                TestLogEvent.SKIPPED,
+                TestLogEvent.STANDARD_OUT
+        exceptionFormat TestExceptionFormat.FULL
+        showExceptions true
+        showCauses true
+        showStackTraces true
+
+        // set options for log level DEBUG and INFO
+        debug {
+
+            events TestLogEvent.STARTED,
+                    TestLogEvent.FAILED,
+                    TestLogEvent.PASSED,
+                    TestLogEvent.SKIPPED,
+                    TestLogEvent.STANDARD_ERROR,
+                    TestLogEvent.STANDARD_OUT
+            exceptionFormat TestExceptionFormat.FULL
+        }
+        info.events = debug.events
+        info.exceptionFormat = debug.exceptionFormat
+
+        afterSuite { desc, result ->
+            if (!desc.parent) { // will match the outermost suite
+                def output = "Results: ${result.resultType} (${result.testCount} tests, ${result.successfulTestCount} successes, ${result.failedTestCount} failures, ${result.skippedTestCount} skipped)"
+                def startItem = '|  ', endItem = '  |'
+                def repeatLength = startItem.length() + output.length() + endItem.length()
+                println('\n' + ('-' * repeatLength) + '\n' + startItem + output + endItem + '\n' + ('-' * repeatLength))
+            }
+        }
+    }
+}
+
 jacocoTestReport {
     reports {
         xml.enabled = true

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-all.zip
diff --git a/jitpack.yml b/jitpack.yml
@@ -1,3 +1,2 @@
 jdk:
- - oraclejdk9
-
+ - oraclejdk11
diff --git a/src/main/java/com/upserve/uppend/AppendOnlyStore.java b/src/main/java/com/upserve/uppend/AppendOnlyStore.java
@@ -1,10 +1,5 @@
 package com.upserve.uppend;
 
-import com.github.benmanes.caffeine.cache.stats.CacheStats;
-import com.upserve.uppend.lookup.FlushStats;
-
-import java.io.Flushable;
-
 /**
  * Add byte arrays under a key and partition, and retrieve them. Note the
  * expectation that the byte arrays are appended to the value, which is an
@@ -46,6 +41,4 @@ public interface AppendOnlyStore extends ReadOnlyAppendOnlyStore, RegisteredFlus
      * @return the name
      */
     String getName();
-
-    FlushStats getFlushStats();
 }
diff --git a/src/main/java/com/upserve/uppend/AppendOnlyStoreBuilder.java b/src/main/java/com/upserve/uppend/AppendOnlyStoreBuilder.java
@@ -1,55 +1,30 @@
 package com.upserve.uppend;
 
-import com.upserve.uppend.blobs.PageCache;
 import com.upserve.uppend.metrics.AppendOnlyStoreWithMetrics;
 
-import java.util.concurrent.*;
-
 public class AppendOnlyStoreBuilder extends FileStoreBuilder<AppendOnlyStoreBuilder> {
-
     // Blocked Longs Config Options
     public static final int DEFAULT_BLOBS_PER_BLOCK = 127;
 
     private int blobsPerBlock = DEFAULT_BLOBS_PER_BLOCK;
 
     // Blob Cache Options
     public static final int DEFAULT_BLOB_PAGE_SIZE = 4 * 1024 * 1024;
-    public static final int DEFAULT_MAXIMUM_CACHED_BLOB_PAGES = 1024;
-    public static final int DEFAULT_INITIAL_BLOB_PAGE_CACHE_SIZE = 256;
 
     private int blobPageSize = DEFAULT_BLOB_PAGE_SIZE;
-    private int maximumCachedBlobPages = DEFAULT_MAXIMUM_CACHED_BLOB_PAGES;
-    private int initialBlobPageCacheSize = DEFAULT_INITIAL_BLOB_PAGE_CACHE_SIZE;
-
-    private ExecutorService blobCacheExecutorService = ForkJoinPool.commonPool();
 
     // Blocked Long Options
     public AppendOnlyStoreBuilder withBlobsPerBlock(int blobsPerBlock) {
         this.blobsPerBlock = blobsPerBlock;
         return this;
     }
 
-    // Blob Cache Options
+    // Blob Options
     public AppendOnlyStoreBuilder withBlobPageSize(int blobPageSize) {
         this.blobPageSize = blobPageSize;
         return this;
     }
 
-    public AppendOnlyStoreBuilder withMaximumBlobCacheSize(int maximumBlobCacheSize) {
-        this.maximumCachedBlobPages = maximumBlobCacheSize;
-        return this;
-    }
-
-    public AppendOnlyStoreBuilder withInitialBlobCacheSize(int initialBlobCacheSize) {
-        this.initialBlobPageCacheSize = initialBlobCacheSize;
-        return this;
-    }
-
-    public AppendOnlyStoreBuilder withBlobCacheExecutorService(ExecutorService blobCacheExecutorService) {
-        this.blobCacheExecutorService = blobCacheExecutorService;
-        return this;
-    }
-
     public AppendOnlyStore build() {
         return build(false);
     }
@@ -64,16 +39,6 @@ public ReadOnlyAppendOnlyStore buildReadOnly() {
         return build(true);
     }
 
-    public PageCache buildBlobPageCache(String metricsPrefix) {
-        return new PageCache(
-                getBlobPageSize(),
-                getInitialBlobPageCacheSize(),
-                getMaximumCachedBlobPages(),
-                getBlobCacheExecutorService(),
-                metricsSupplier(metricsPrefix, BLOB_PAGE_CACHE_METRICS)
-        );
-    }
-
     public int getBlobsPerBlock() {
         return blobsPerBlock;
     }
@@ -82,49 +47,11 @@ public int getBlobPageSize() {
         return blobPageSize;
     }
 
-    public int getMaximumCachedBlobPages() {
-        return maximumCachedBlobPages;
-    }
-
-    public int getInitialBlobPageCacheSize() {
-        return initialBlobPageCacheSize;
-    }
-
-    public ExecutorService getBlobCacheExecutorService() {
-        return blobCacheExecutorService;
-    }
-
     @Override
     public String toString() {
         return "AppendOnlyStoreBuilder{" +
                 "blobsPerBlock=" + blobsPerBlock +
                 ", blobPageSize=" + blobPageSize +
-                ", maximumCachedBlobPages=" + maximumCachedBlobPages +
-                ", initialBlobPageCacheSize=" + initialBlobPageCacheSize +
-                ", blobCacheExecutorService=" + blobCacheExecutorService +
-                ", storeName='" + storeName + '\'' +
-                ", partitionSize=" + partitionSize +
-                ", lookupHashSize=" + lookupHashSize +
-                ", lookupPageSize=" + lookupPageSize +
-                ", initialLookupPageCacheSize=" + initialLookupPageCacheSize +
-                ", maximumLookupPageCacheSize=" + maximumLookupPageCacheSize +
-                ", maximumLookupKeyCacheWeight=" + maximumLookupKeyCacheWeight +
-                ", initialLookupKeyCacheSize=" + initialLookupKeyCacheSize +
-                ", maximumMetaDataCacheWeight=" + maximumMetaDataCacheWeight +
-                ", initialMetaDataCacheSize=" + initialMetaDataCacheSize +
-                ", metadataTTL=" + metadataTTL +
-                ", metaDataPageSize=" + metaDataPageSize +
-                ", lookupKeyCacheExecutorService=" + lookupKeyCacheExecutorService +
-                ", lookupMetaDataCacheExecutorService=" + lookupMetaDataCacheExecutorService +
-                ", lookupPageCacheExecutorService=" + lookupPageCacheExecutorService +
-                ", flushDelaySeconds=" + flushDelaySeconds +
-                ", flushThreshold=" + flushThreshold +
-                ", dir=" + dir +
-                ", storeMetricsRegistry=" + storeMetricsRegistry +
-                ", metricsRootName='" + metricsRootName + '\'' +
-                ", storeMetrics=" + storeMetrics +
-                ", cacheMetricsRegistry=" + cacheMetricsRegistry +
-                ", cacheMetrics=" + cacheMetrics +
-                '}';
+                '}' + super.toString();
     }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,3 +2,4 @@ @@
     build/
     .idea/
     *.iml
+    logs/