Merge pull request #101 from upserve/docs

Docs, cache size and formatting Merging with failed Travis build - 403 on Maven dependencies.
upserve · Aug 9, 2019 · d9ddb49 · d9ddb49
2 parents f29374a + 459668f
commit d9ddb49
Show file tree

Hide file tree

Showing 8 changed files with 38 additions and 31 deletions.
diff --git a/README.md b/README.md
@@ -3,6 +3,7 @@ Uppend: an append-only, key-multivalue store
 [![Build Status](https://img.shields.io/travis/upserve/uppend/master.svg?style=flat-square)](https://travis-ci.org/upserve/uppend)
 [![Release Artifact](https://img.shields.io/maven-central/v/com.upserve/uppend.svg?style=flat-square)](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.upserve%22%20AND%20a%3Auppend)
 [![Test Coverage](https://img.shields.io/codecov/c/github/upserve/uppend/master.svg?style=flat-square)](https://codecov.io/github/upserve/uppend?branch=master)
+[![Release](https://jitpack.io/v/upserve/uppend.svg)](https://jitpack.io/#upserve/Uppend)
 
 Uppend is an append-only, key-multivalue store which is suitable for streaming
 event aggregation. It assumes a single writer process, and appended values are
@@ -17,13 +18,13 @@ Maven:
 <dependency>
     <groupId>com.upserve</groupId>
     <artifactId>uppend</artifactId>
-    <version>0.0.1</version>
+    <version>0.2.1</version>
 </dependency>
 ```
 
 Gradle:
 ```gradle
-compile 'com.upserve:uppend:0.0.1'
+compile 'com.upserve:uppend:0.2.1'
 ```
 
 Hello world:
@@ -67,12 +68,24 @@ To run tests in a specific path
 ```
 
 Example script to fork the benchmark with a system resource monitor like IOSTAT
+
+_runtest.sh_
 ```sh
 trap "kill 0" EXIT
 
-java -jar build/libs/uppend-all-0.0.2-91-g6abbf45.dirty.jar benchmark ../foo/test & BENCHMARK_PID=$!
-iostat -d 2 & IOSTAT_PID=$!
+java -Xmx32g -jar ./uppend-all-0.2.1.jar benchmark -c $C -m $M -s $S -b $B ./data1.output & BENCHMARK_PID=$!
+
+iostat -c -d 5 -x & IOSTAT_PID=$!
 
 wait $BENCHMARK_PID
 kill $IOSTAT_PID
+```
+
+Call _runtest.sh_ with:
+```sh
+export C=wide
+export M=read
+export S=large
+export B=medium
+./runtest.sh 2>&1 | tee /mnt/log/${M}_${C}_${S}_${B}.log
 ```
diff --git a/build.gradle b/build.gradle
@@ -1,6 +1,8 @@
 buildscript {
     repositories {
-        jcenter()
+//        jcenter()
+//        mavenCentral()
+        google()
     }
     dependencies {
         classpath 'com.bmuschko:gradle-nexus-plugin:2.3.1'

diff --git a/src/main/java/com/upserve/uppend/lookup/LookupMetadata.java b/src/main/java/com/upserve/uppend/lookup/LookupMetadata.java
@@ -4,11 +4,9 @@
 import com.upserve.uppend.metrics.LookupDataMetrics;
 import org.slf4j.Logger;
 
-import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.nio.*;
 import java.util.*;
-import java.util.concurrent.atomic.LongAdder;
 /**
  * The bisect tree is linearized as follows
  *              8
@@ -32,7 +30,8 @@
 public class LookupMetadata {
     private static final Logger log = org.slf4j.LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-    private static final int MAX_BISECT_KEY_CACHE_DEPTH = 9; // Size == 1024
+    // TODO make MAX_BISECT_KEY_CACHE_DEPTH configurable via the FileStoreBuilder
+    private static final int MAX_BISECT_KEY_CACHE_DEPTH = 13; // Size == 16_384
     private static final int MAX_TREE_NODES = treeSize(MAX_BISECT_KEY_CACHE_DEPTH);
     private final LookupKey[] bisectKeys = new LookupKey[MAX_TREE_NODES];
 
@@ -43,15 +42,8 @@ public class LookupMetadata {
     private final LookupKey maxKey;
     private final int[] keyStorageOrder;
 
-    final LookupDataMetrics.Adders lookupDataMetricsAdders;
-    final byte[] checksum;
-
-    static LookupMetadata generateMetadata(LookupKey minKey, LookupKey maxKey, int[] keyStorageOrder,
-                                                  VirtualMutableBlobStore metaDataBlobs, int metadataGeneration) {
-        return generateMetadata(
-                minKey, maxKey, keyStorageOrder, metaDataBlobs, metadataGeneration, new LookupDataMetrics.Adders()
-        );
-    }
+    private final LookupDataMetrics.Adders lookupDataMetricsAdders;
+    private final byte[] checksum;
 
     static LookupMetadata generateMetadata(LookupKey minKey, LookupKey maxKey, int[] keyStorageOrder,
                                                   VirtualMutableBlobStore metaDataBlobs, int metadataGeneration,
@@ -74,7 +66,7 @@ static LookupMetadata generateMetadata(LookupKey minKey, LookupKey maxKey, int[]
 
     }
 
-    LookupMetadata(LookupKey minKey, LookupKey maxKey, int[] keyStorageOrder, int metadataGeneration,
+    private LookupMetadata(LookupKey minKey, LookupKey maxKey, int[] keyStorageOrder, int metadataGeneration,
                    LookupDataMetrics.Adders lookupDataMetricsAdders) {
         this.numKeys = keyStorageOrder.length;
         this.minKey = minKey;
@@ -251,15 +243,15 @@ Long findKey(VirtualLongBlobStore longBlobStore, LookupKey key) {
         }
     }
 
-    public static int treeSize(int depth) {
+    private static int treeSize(int depth) {
         return 1 << (depth +1);
     }
 
     void clearLookupTree(){
         Arrays.fill(bisectKeys, null);
     }
 
-    public void writeTo(VirtualMutableBlobStore metadataBlobs) {
+    void writeTo(VirtualMutableBlobStore metadataBlobs) {
         int headerSize = 12 + minKey.byteLength() + maxKey.byteLength();
         int intBufSize = 4 * numKeys;
         ByteBuffer byteBuffer = ByteBuffer.allocate(headerSize + intBufSize);
@@ -285,23 +277,23 @@ public String toString() {
                 '}';
     }
 
-    public int getMetadataGeneration() {
+    int getMetadataGeneration() {
         return metadataGeneration;
     }
 
-    public int getNumKeys() {
+    int getNumKeys() {
         return numKeys;
     }
 
-    public int[] getKeyStorageOrder() {
+    int[] getKeyStorageOrder() {
         return keyStorageOrder;
     }
 
-    public LookupKey getMinKey() {
+    LookupKey getMinKey() {
         return minKey;
     }
 
-    public LookupKey getMaxKey() {
+    LookupKey getMaxKey() {
         return maxKey;
     }
 }
diff --git a/src/main/java/com/upserve/uppend/metrics/BlobStoreMetrics.java b/src/main/java/com/upserve/uppend/metrics/BlobStoreMetrics.java
@@ -35,7 +35,7 @@ public BlobStoreMetrics(Adders blobStoreMetricsAdders, LongSummaryStatistics blo
                 blobStoreMetricsAdders.readCounter.sum(),
                 blobStoreMetricsAdders.readTimer.sum(),
                 blobStoreAllocatedPagesStatistics.getAverage(),
-                blobStoreAllocatedPagesStatistics.getMax(),
+                Math.max(blobStoreAllocatedPagesStatistics.getMax(), 0),
                 blobStoreAllocatedPagesStatistics.getSum()
         );
     }

diff --git a/src/main/java/com/upserve/uppend/metrics/BlockedLongMetrics.java b/src/main/java/com/upserve/uppend/metrics/BlockedLongMetrics.java
@@ -48,11 +48,11 @@ public BlockedLongMetrics(Adders blockedLongMetricsAdders, LongSummaryStatistics
                 blockedLongMetricsAdders.readLastTimer.sum(),
 
                 blockedLongAllocatedBlocksStatistics.getAverage(),
-                blockedLongAllocatedBlocksStatistics.getMax(),
+                Math.max(blockedLongAllocatedBlocksStatistics.getMax(), 0),
                 blockedLongAllocatedBlocksStatistics.getSum(),
 
                 blockedLongAppendCountStatistics.getAverage(),
-                blockedLongAppendCountStatistics.getMax(),
+                Math.max(blockedLongAppendCountStatistics.getMax(), 0),
                 blockedLongAppendCountStatistics.getSum()
         );
     }

diff --git a/src/main/java/com/upserve/uppend/metrics/LongBlobStoreMetrics.java b/src/main/java/com/upserve/uppend/metrics/LongBlobStoreMetrics.java
@@ -57,7 +57,7 @@ public LongBlobStoreMetrics(Adders blobStoreMetricsAdders, LongSummaryStatistics
                 blobStoreMetricsAdders.longReadTimer.sum(),
 
                 longblobStoreAllocatedPagesStatistics.getAverage(),
-                longblobStoreAllocatedPagesStatistics.getMax(),
+                Math.max(longblobStoreAllocatedPagesStatistics.getMax(), 0),
                 longblobStoreAllocatedPagesStatistics.getSum()
         );
     }

diff --git a/src/main/java/com/upserve/uppend/metrics/LookupDataMetrics.java b/src/main/java/com/upserve/uppend/metrics/LookupDataMetrics.java
@@ -45,7 +45,7 @@ public LookupDataMetrics(Adders lookupDataMetricsAdders, LongSummaryStatistics l
                 lookupDataMetricsAdders.cacheHitCount.sum(),
                 lookupDataMetricsAdders.findKeyTimer.sum(),
                 lookupDataSizeStatistics.getAverage(),
-                lookupDataSizeStatistics.getMax(),
+                Math.max(lookupDataSizeStatistics.getMax(), 0),
                 lookupDataSizeStatistics.getSum()
         );
     }

diff --git a/src/main/java/com/upserve/uppend/metrics/MutableBlobStoreMetrics.java b/src/main/java/com/upserve/uppend/metrics/MutableBlobStoreMetrics.java
@@ -35,7 +35,7 @@ public MutableBlobStoreMetrics(Adders mutableBlobStoreMetricsAdders, LongSummary
                 mutableBlobStoreMetricsAdders.readCounter.sum(),
                 mutableBlobStoreMetricsAdders.readTimer.sum(),
                 mutableStoreAllocatedPagesStatistics.getAverage(),
-                mutableStoreAllocatedPagesStatistics.getMax(),
+                Math.max(mutableStoreAllocatedPagesStatistics.getMax(), 0),
                 mutableStoreAllocatedPagesStatistics.getSum()
         );
     }