Skip to content

Commit

Permalink
Merge pull request #100 from upserve/fixed_lookupmetadata_hit_miss_co…
Browse files Browse the repository at this point in the history
…unts

Fixed LookupMetadata hit/miss counts
  • Loading branch information
dstuebe authored Aug 8, 2019
2 parents a03663e + 9e15002 commit f29374a
Show file tree
Hide file tree
Showing 36 changed files with 1,775 additions and 686 deletions.
9 changes: 8 additions & 1 deletion src/main/java/com/upserve/uppend/AppendOnlyStoreBuilder.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.upserve.uppend;

import com.upserve.uppend.metrics.AppendOnlyStoreWithMetrics;
import com.upserve.uppend.metrics.*;

public class AppendOnlyStoreBuilder extends FileStoreBuilder<AppendOnlyStoreBuilder> {
// Blocked Longs Config Options
Expand All @@ -13,6 +13,9 @@ public class AppendOnlyStoreBuilder extends FileStoreBuilder<AppendOnlyStoreBuil

private int blobPageSize = DEFAULT_BLOB_PAGE_SIZE;

private BlobStoreMetrics.Adders blobStoreMetricsAdders = new BlobStoreMetrics.Adders();
private BlockedLongMetrics.Adders blockedLongMetricsAdders = new BlockedLongMetrics.Adders();

// Blocked Long Options
public AppendOnlyStoreBuilder withBlobsPerBlock(int blobsPerBlock) {
this.blobsPerBlock = blobsPerBlock;
Expand Down Expand Up @@ -47,6 +50,10 @@ public int getBlobPageSize() {
return blobPageSize;
}

public BlobStoreMetrics.Adders getBlobStoreMetricsAdders() { return blobStoreMetricsAdders; }

public BlockedLongMetrics.Adders getBlockedLongMetricsAdders() { return blockedLongMetricsAdders; }

@Override
public String toString() {
return "AppendOnlyStoreBuilder{" +
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/com/upserve/uppend/AppendStoreMetrics.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package com.upserve.uppend;

import com.upserve.uppend.metrics.*;

public interface AppendStoreMetrics {
BlockedLongMetrics getBlockedLongMetrics();
BlobStoreMetrics getBlobStoreMetrics();
}
148 changes: 100 additions & 48 deletions src/main/java/com/upserve/uppend/AppendStorePartition.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,60 +28,108 @@ private static Path blocksFile(Path partitiondDir) {
return partitiondDir.resolve("blockedLongs");
}

public static AppendStorePartition createPartition(Path parentDir, String partition, int hashCount, int targetBufferSize, int flushThreshold, int reloadInterval, int metadataPageSize, int blockSize, int blobPageSize, int keyPageSize) {
Path partitionDir = validatePartition(parentDir, partition);

BlockedLongs blocks = new BlockedLongs(blocksFile(partitionDir), blockSize, false);
VirtualPageFile blobs = new VirtualPageFile(blobsFile(partitionDir), hashCount, blobPageSize, targetBufferSize,false);
VirtualPageFile metadata = new VirtualPageFile(metadataPath(partitionDir), hashCount, metadataPageSize, adjustedTargetBufferSize(metadataPageSize, hashCount, targetBufferSize),false);
VirtualPageFile keys = new VirtualPageFile(keysPath(partitionDir), hashCount, keyPageSize, adjustedTargetBufferSize(keyPageSize, hashCount, targetBufferSize),false);
static AppendStorePartition createPartition(Path parentDir, String partition, AppendOnlyStoreBuilder builder) {

return new AppendStorePartition(keys, metadata, blobs, blocks, hashCount, flushThreshold, reloadInterval, false);
}
Path partitionDir = validatePartition(parentDir, partition);

public static AppendStorePartition openPartition(Path parentDir, String partition, int hashCount, int targetBufferSize, int flushThreshold, int reloadInterval, int metadataPageSize, int blockSize, int blobPageSize, int keyPageSize, boolean readOnly) {
BlockedLongs blocks = new BlockedLongs(
blocksFile(partitionDir),
builder.getBlobsPerBlock(),
false,
builder.getBlockedLongMetricsAdders()
);
VirtualPageFile blobs = new VirtualPageFile(
blobsFile(partitionDir),
builder.getLookupHashCount(),
builder.getBlobPageSize(),
builder.getTargetBufferSize(),
false
);
VirtualPageFile metadata = new VirtualPageFile(
metadataPath(partitionDir),
builder.getLookupHashCount(),
builder.getMetadataPageSize(),
adjustedTargetBufferSize(
builder.getMetadataPageSize(),
builder.getLookupHashCount(),
builder.getTargetBufferSize()),
false
);
VirtualPageFile keys = new VirtualPageFile(
keysPath(partitionDir),
builder.getLookupHashCount(),
builder.getLookupPageSize(),
adjustedTargetBufferSize(
builder.getLookupPageSize(),
builder.getLookupHashCount(),
builder.getTargetBufferSize()
),
false
);

return new AppendStorePartition(keys, metadata, blobs, blocks, false, builder);
}

static AppendStorePartition openPartition(Path parentDir, String partition, boolean readOnly, AppendOnlyStoreBuilder builder) {
validatePartition(partition);
Path partitionDir = parentDir.resolve(partition);

if (!(Files.exists(blocksFile(partitionDir)) && Files.exists(metadataPath(partitionDir))
&& Files.exists(keysPath(partitionDir)) && Files.exists(blobsFile(partitionDir)))) return null;

BlockedLongs blocks = new BlockedLongs(blocksFile(partitionDir), blockSize, readOnly);

VirtualPageFile blobs = new VirtualPageFile(blobsFile(partitionDir), hashCount, blobPageSize, targetBufferSize, readOnly);
VirtualPageFile metadata = new VirtualPageFile(metadataPath(partitionDir), hashCount, metadataPageSize, adjustedTargetBufferSize(metadataPageSize, hashCount, targetBufferSize), readOnly);
VirtualPageFile keys = new VirtualPageFile(keysPath(partitionDir), hashCount, keyPageSize, adjustedTargetBufferSize(keyPageSize, hashCount, targetBufferSize), readOnly);

return new AppendStorePartition(keys, metadata, blobs, blocks, hashCount, flushThreshold, reloadInterval, readOnly);
}

PartitionStats getPartitionStats(){
LongSummaryStatistics metadataStats = Arrays.stream(lookups)
.mapToLong(LookupData::getMetadataSize)
//.filter(val -> val > 0)
.summaryStatistics();

return new PartitionStats(metadataBlobFile.getAllocatedPageCount(),
longKeyFile.getAllocatedPageCount(),
blobFile.getAllocatedPageCount(),
Arrays.stream(lookups).mapToLong(LookupData::getMetadataLookupMissCount).sum(),
Arrays.stream(lookups).mapToLong(LookupData::getMetadataLookupHitCount).sum(),
metadataStats.getSum(),
Arrays.stream(lookups).mapToLong(LookupData::getFindKeyTimer).sum(),
Arrays.stream(lookups).mapToLong(LookupData::getFlushedKeyCount).sum(),
Arrays.stream(lookups).mapToLong(LookupData::getFlushCount).sum(),
metadataStats.getCount(),
metadataStats.getMax()
);
}

private AppendStorePartition(VirtualPageFile longKeyFile, VirtualPageFile metadataBlobFile, VirtualPageFile blobsFile, BlockedLongs blocks, int hashCount, int flushThreshold, int reloadInterval, boolean readOnly) {
super(longKeyFile, metadataBlobFile, hashCount, flushThreshold, reloadInterval, readOnly);
BlockedLongs blocks = new BlockedLongs(
blocksFile(partitionDir),
builder.getBlobsPerBlock(),
readOnly,
builder.getBlockedLongMetricsAdders()
);

VirtualPageFile blobs = new VirtualPageFile(
blobsFile(partitionDir),
builder.getLookupHashCount(),
builder.getBlobPageSize(),
builder.getTargetBufferSize(),
readOnly
);
VirtualPageFile metadata = new VirtualPageFile(
metadataPath(partitionDir),
builder.getLookupHashCount(),
builder.getMetadataPageSize(),
adjustedTargetBufferSize(
builder.getMetadataPageSize(),
builder.getLookupHashCount(),
builder.getTargetBufferSize()
),
readOnly
);
VirtualPageFile keys = new VirtualPageFile(
keysPath(partitionDir),
builder.getLookupHashCount(),
builder.getLookupPageSize(),
adjustedTargetBufferSize(
builder.getLookupPageSize(),
builder.getLookupHashCount(),
builder.getTargetBufferSize()
),
readOnly
);

return new AppendStorePartition(keys, metadata, blobs, blocks, readOnly, builder);
}

private AppendStorePartition(
VirtualPageFile longKeyFile, VirtualPageFile metadataBlobFile, VirtualPageFile blobsFile,
BlockedLongs blocks, boolean readOnly, AppendOnlyStoreBuilder builder) {
super(longKeyFile, metadataBlobFile, readOnly, builder);

this.blocks = blocks;
this.blobFile = blobsFile;

blobs = IntStream.range(0, hashCount)
.mapToObj(virtualFileNumber -> new VirtualAppendOnlyBlobStore(virtualFileNumber, blobsFile))
.mapToObj(virtualFileNumber -> new VirtualAppendOnlyBlobStore(
virtualFileNumber, blobsFile, builder.getBlobStoreMetricsAdders()
)
)
.toArray(VirtualAppendOnlyBlobStore[]::new);
}

Expand Down Expand Up @@ -147,13 +195,9 @@ Stream<String> keys() {
.flatMap(virtualFileNumber -> lookups[virtualFileNumber].keys().map(LookupKey::string));
}

BlockStats blockedLongStats() {
return blocks.stats();
}

void clear() throws IOException {
longKeyFile.close();
metadataBlobFile.close();
getLongKeyFile().close();
getMetadataBlobFile().close();
blobFile.close();
blocks.close();

Expand All @@ -167,4 +211,12 @@ public void close() throws IOException {
blobFile.close();
blocks.close();
}

BlockedLongs getBlocks() {
return blocks;
}

VirtualPageFile getBlobFile() {
return blobFile;
}
}
74 changes: 0 additions & 74 deletions src/main/java/com/upserve/uppend/BlockStats.java

This file was deleted.

Loading

0 comments on commit f29374a

Please sign in to comment.