Skip to content

Commit

Permalink
PrjHub issue orientechnologies#7760 . Tool to estimate index space us…
Browse files Browse the repository at this point in the history 
…age was added.
  • Loading branch information
@andrii0lomakin
andrii0lomakin committed Oct 25, 2016
1 parent 4b79412 commit 09eca37
Show file tree
Hide file tree
Showing 15 changed files with 293 additions and 115 deletions.
9 changes: 9 additions & 0 deletions core/src/main/java/com/orientechnologies/orient/core/index/OIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,4 +312,13 @@ OIndex<T> create(String name, OIndexDefinition indexDefinition, String clusterIn
Object getFirstKey();

Object getLastKey();

/**
* Average percent of usage of space in index pages.
* Two values are returned. First value indicates usage of space in leaf pages, second value indicates usage of space in
* non-leaf pages. Not all indexes and storages support this method.
*
* @return Average percent of usage of space in index pages.
*/
int[] spaceUsage();
}
18 changes: 14 additions & 4 deletions core/src/main/java/com/orientechnologies/orient/core/index/OIndexAbstract.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,10 @@
*/
public abstract class OIndexAbstract<T> implements OIndexInternal<T>, OOrientStartupListener, OOrientShutdownListener {
protected static final String CONFIG_CLUSTERS = "clusters";
protected final OIndexEngine<T> indexEngine;
private final String databaseName;
protected String type;
protected String valueContainerAlgorithm;
protected final OIndexEngine<T> indexEngine;
private final String databaseName;
protected String type;
protected String valueContainerAlgorithm;
protected final ONewLockManager<Object> keyLockManager = new ONewLockManager<Object>(
OGlobalConfiguration.ENVNRONMENT_CONCURRENCY_LEVEL.getValueAsInteger());

Expand Down Expand Up @@ -371,6 +371,16 @@ public Object getLastKey() {
}
}

@Override
public int[] spaceUsage() {
acquireSharedLock();
try {
return indexEngine.spaceUsage();
} finally {
releaseSharedLock();
}
}

/**
* {@inheritDoc}
*/
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/com/orientechnologies/orient/core/index/OIndexAbstractDelegate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ public OIndex<T> clear() {
return delegate.clear();
}

@Override
public int[] spaceUsage() {
return delegate.spaceUsage();
}

@Override
public OIndexCursor iterateEntriesBetween(Object fromKey, boolean fromInclusive, Object toKey, boolean toInclusive,
boolean ascOrder) {
Expand Down
10 changes: 9 additions & 1 deletion core/src/main/java/com/orientechnologies/orient/core/index/OIndexEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import java.util.Collection;

import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.id.ORID;
import com.orientechnologies.orient.core.serialization.serializer.stream.OStreamSerializer;

/**
Expand Down Expand Up @@ -79,6 +78,15 @@ OIndexCursor iterateEntriesMinor(final Object toKey, final boolean isInclusive,

int getVersion();

/**
* Average percent of usage of space in index pages.
* Two values are returned. First value indicates usage of space in leaf pages, second value indicates usage of space in
* non-leaf pages. Not all indexes and storages support this method.
*
* @return Average percent of usage of space in index pages.
*/
int[] spaceUsage();

interface ValuesTransformer<V> {
Collection<OIdentifiable> transformFromValue(V value);
}
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/com/orientechnologies/orient/core/index/OIndexRecorder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,11 @@ public OIndex<OIdentifiable> create(String name, OIndexDefinition indexDefinitio

}

@Override
public int[] spaceUsage() {
throw new UnsupportedOperationException("Unsupported operation");
}

@Override
public String getDatabaseName() {
return delegate.getDatabaseName();
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/com/orientechnologies/orient/core/index/OIndexRemote.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,11 @@ public long count(final Object iRangeFrom, final boolean iFromInclusive, final O
return (Long) getDatabase().command(cmd).execute(iRangeFrom, iRangeTo);
}

@Override
public int[] spaceUsage() {
throw new UnsupportedOperationException("Unsupported operation");
}

public OIndexRemote<T> put(final Object iKey, final OIdentifiable iValue) {
if (iValue instanceof ORecord && !iValue.getIdentity().isValid())
// SAVE IT BEFORE TO PUT
Expand Down
43 changes: 25 additions & 18 deletions core/src/main/java/com/orientechnologies/orient/core/index/engine/OHashTableIndexEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,17 +44,17 @@
* @since 15.07.13
*/
public final class OHashTableIndexEngine<V> implements OIndexEngine<V> {
public static final int VERSION = 2;
public static final int VERSION = 2;

public static final String METADATA_FILE_EXTENSION = ".him";
public static final String TREE_FILE_EXTENSION = ".hit";
public static final String BUCKET_FILE_EXTENSION = ".hib";
public static final String NULL_BUCKET_FILE_EXTENSION = ".hnb";
public static final String METADATA_FILE_EXTENSION = ".him";
public static final String TREE_FILE_EXTENSION = ".hit";
public static final String BUCKET_FILE_EXTENSION = ".hib";
public static final String NULL_BUCKET_FILE_EXTENSION = ".hnb";

private final OHashTable<Object, V> hashTable;
private final OMurmurHash3HashFunction<Object> hashFunction;

private int version;
private int version;

public OHashTableIndexEngine(String name, Boolean durableInNonTxMode, OAbstractPaginatedStorage storage, int version) {
hashFunction = new OMurmurHash3HashFunction<Object>();
Expand Down Expand Up @@ -97,15 +97,21 @@ public void create(OIndexDefinition indexDefinition, String clusterIndexName, OS
keySerializer = new OSimpleKeySerializer();

hashFunction.setValueSerializer(keySerializer);
hashTable.create(keySerializer, (OBinarySerializer<V>) valueSerializer, indexDefinition != null ? indexDefinition.getTypes()
: null, indexDefinition != null && !indexDefinition.isNullValuesIgnored());
hashTable
.create(keySerializer, (OBinarySerializer<V>) valueSerializer, indexDefinition != null ? indexDefinition.getTypes() : null,
indexDefinition != null && !indexDefinition.isNullValuesIgnored());
}

@Override
public void flush() {
hashTable.flush();
}

@Override
public int[] spaceUsage() {
throw new UnsupportedOperationException("Unsupported operation");
}

@Override
public void deleteWithoutLoad(String indexName) {
hashTable.deleteWithoutLoad(indexName, (OAbstractPaginatedStorage) getDatabase().getStorage().getUnderlying());
Expand All @@ -118,8 +124,8 @@ public void delete() {

@Override
public void load(String indexName, OIndexDefinition indexDefinition, OStreamSerializer valueSerializer, boolean isAutomatic) {
hashTable.load(indexName, indexDefinition != null ? indexDefinition.getTypes() : null, indexDefinition != null
&& !indexDefinition.isNullValuesIgnored());
hashTable.load(indexName, indexDefinition != null ? indexDefinition.getTypes() : null,
indexDefinition != null && !indexDefinition.isNullValuesIgnored());
hashFunction.setValueSerializer(hashTable.getKeySerializer());
}

Expand Down Expand Up @@ -199,7 +205,8 @@ public OIndexCursor iterateEntriesMajor(Object fromKey, boolean isInclusive, boo
}

@Override
public OIndexCursor iterateEntriesMinor(Object toKey, boolean isInclusive, boolean ascSortOrder, ValuesTransformer<V> transformer) {
public OIndexCursor iterateEntriesMinor(Object toKey, boolean isInclusive, boolean ascSortOrder,
ValuesTransformer<V> transformer) {
throw new UnsupportedOperationException("iterateEntriesMinor");
}

Expand All @@ -216,11 +223,11 @@ public Object getLastKey() {
@Override
public OIndexCursor cursor(final ValuesTransformer<V> valuesTransformer) {
return new OIndexAbstractCursor() {
private int nextEntriesIndex;
private int nextEntriesIndex;
private OHashIndexBucket.Entry<Object, V>[] entries;

private Iterator<OIdentifiable> currentIterator = new OEmptyIterator<OIdentifiable>();
private Object currentKey;
private Iterator<OIdentifiable> currentIterator = new OEmptyIterator<OIdentifiable>();
private Object currentKey;

{
OHashIndexBucket.Entry<Object, V> firstEntry = hashTable.firstEntry();
Expand Down Expand Up @@ -299,11 +306,11 @@ public OIdentifiable setValue(OIdentifiable value) {
@Override
public OIndexCursor descCursor(final ValuesTransformer<V> valuesTransformer) {
return new OIndexAbstractCursor() {
private int nextEntriesIndex;
private int nextEntriesIndex;
private OHashIndexBucket.Entry<Object, V>[] entries;

private Iterator<OIdentifiable> currentIterator = new OEmptyIterator<OIdentifiable>();
private Object currentKey;
private Iterator<OIdentifiable> currentIterator = new OEmptyIterator<OIdentifiable>();
private Object currentKey;

{
OHashIndexBucket.Entry<Object, V> lastEntry = hashTable.lastEntry();
Expand Down Expand Up @@ -382,7 +389,7 @@ public OIdentifiable setValue(OIdentifiable value) {
@Override
public OIndexKeyCursor keyCursor() {
return new OIndexKeyCursor() {
private int nextEntriesIndex;
private int nextEntriesIndex;
private OHashIndexBucket.Entry<Object, V>[] entries;

{
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/com/orientechnologies/orient/core/index/engine/ORemoteIndexEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,11 @@ public OIndexKeyCursor keyCursor() {
throw new UnsupportedOperationException("keyCursor");
}

@Override
public int[] spaceUsage() {
throw new UnsupportedOperationException("Unsupported operation");
}

@Override
public int getVersion() {
return -1;
Expand Down
39 changes: 25 additions & 14 deletions core/src/main/java/com/orientechnologies/orient/core/index/engine/OSBTreeIndexEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,17 @@
* @since 8/30/13
*/
public class OSBTreeIndexEngine<V> extends OSharedResourceAdaptiveExternal implements OIndexEngine<V> {
public static final int VERSION = 1;
public static final int VERSION = 1;

public static final String DATA_FILE_EXTENSION = ".sbt";
public static final String NULL_BUCKET_FILE_EXTENSION = ".nbt";
public static final String DATA_FILE_EXTENSION = ".sbt";
public static final String NULL_BUCKET_FILE_EXTENSION = ".nbt";

private final OSBTree<Object, V> sbTree;
private int version;
private int version;

public OSBTreeIndexEngine(String name, Boolean durableInNonTxMode, OAbstractPaginatedStorage storage, int version) {
super(OGlobalConfiguration.ENVIRONMENT_CONCURRENT.getValueAsBoolean(), OGlobalConfiguration.MVRBTREE_TIMEOUT
.getValueAsInteger(), true);
super(OGlobalConfiguration.ENVIRONMENT_CONCURRENT.getValueAsBoolean(),
OGlobalConfiguration.MVRBTREE_TIMEOUT.getValueAsInteger(), true);

boolean durableInNonTx;

Expand Down Expand Up @@ -92,8 +92,9 @@ public void create(OIndexDefinition indexDefinition, String clusterIndexName, OS
final OBinarySerializer keySerializer = determineKeySerializer(indexDefinition);
final int keySize = determineKeySize(indexDefinition);

sbTree.create(keySerializer, (OBinarySerializer<V>) valueSerializer, indexDefinition != null ? indexDefinition.getTypes()
: null, keySize, indexDefinition != null && !indexDefinition.isNullValuesIgnored());
sbTree.create(keySerializer, (OBinarySerializer<V>) valueSerializer,
indexDefinition != null ? indexDefinition.getTypes() : null, keySize,
indexDefinition != null && !indexDefinition.isNullValuesIgnored());
} finally {
releaseExclusiveLock();
}
Expand Down Expand Up @@ -152,8 +153,8 @@ public void load(String indexName, OIndexDefinition indexDefinition, OStreamSeri
final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) database.getStorage().getUnderlying();

sbTree.load(indexName, determineKeySerializer(indexDefinition), valueSerializer,
indexDefinition != null ? indexDefinition.getTypes() : null, determineKeySize(indexDefinition), indexDefinition != null
&& !indexDefinition.isNullValuesIgnored());
indexDefinition != null ? indexDefinition.getTypes() : null, determineKeySize(indexDefinition),
indexDefinition != null && !indexDefinition.isNullValuesIgnored());
} finally {
releaseExclusiveLock();
}
Expand All @@ -179,7 +180,6 @@ public boolean remove(Object key) {
}
}


@Override
public int getVersion() {
return version;
Expand Down Expand Up @@ -324,7 +324,18 @@ public OIndexCursor iterateEntriesMajor(Object fromKey, boolean isInclusive, boo
}

@Override
public OIndexCursor iterateEntriesMinor(Object toKey, boolean isInclusive, boolean ascSortOrder, ValuesTransformer<V> transformer) {
public int[] spaceUsage() {
acquireSharedLock();
try {
return sbTree.spaceUsage();
} finally {
releaseSharedLock();
}
}

@Override
public OIndexCursor iterateEntriesMinor(Object toKey, boolean isInclusive, boolean ascSortOrder,
ValuesTransformer<V> transformer) {
acquireSharedLock();
try {
return new OSBTreeIndexCursor<V>(sbTree.iterateEntriesMinor(toKey, isInclusive, ascSortOrder), transformer);
Expand Down Expand Up @@ -377,8 +388,8 @@ private static final class OSBTreeIndexCursor<V> extends OIndexAbstractCursor {
private final OSBTree.OSBTreeCursor<Object, V> treeCursor;
private final ValuesTransformer<V> valuesTransformer;

private Iterator<OIdentifiable> currentIterator = OEmptyIterator.IDENTIFIABLE_INSTANCE;
private Object currentKey = null;
private Iterator<OIdentifiable> currentIterator = OEmptyIterator.IDENTIFIABLE_INSTANCE;
private Object currentKey = null;

private OSBTreeIndexCursor(OSBTree.OSBTreeCursor<Object, V> treeCursor, ValuesTransformer<V> valuesTransformer) {
this.treeCursor = treeCursor;
Expand Down
49 changes: 47 additions & 2 deletions core/src/main/java/com/orientechnologies/orient/core/index/sbtree/local/OSBTree.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,52 @@ public K next(int prefetchSize) {
}
}

public int[] spaceUsage() {
long leafSpaceUsage = 0;
long leafSpaceUsageCounter = 0;

long nonLeafSpaceUsage = 0;
long nonLeafSpaceUsageCounter = 0;

atomicOperationsManager.acquireReadLock(this);
try {
acquireSharedLock();
try {
final OAtomicOperation atomicOperation = atomicOperationsManager.getCurrentOperation();

final long filledUpTo = getFilledUpTo(atomicOperation, fileId);
for (long pageIndex = 0; pageIndex < filledUpTo; pageIndex++) {
final OCacheEntry cacheEntry = loadPage(atomicOperation, fileId, pageIndex, false);
cacheEntry.acquireSharedLock();
try {
OSBTreeBucket<K, V> bucket = new OSBTreeBucket<K, V>(cacheEntry, keySerializer, keyTypes, valueSerializer,
getChangesTree(atomicOperation, cacheEntry));
if (bucket.isLeaf()) {
leafSpaceUsageCounter++;
leafSpaceUsage += bucket.spaceUsage();
} else {
nonLeafSpaceUsageCounter++;
nonLeafSpaceUsage += bucket.spaceUsage();
}
} finally {
cacheEntry.releaseSharedLock();
releasePage(atomicOperation, cacheEntry);
}

}

} finally {
releaseSharedLock();
}
} catch (IOException ioe) {
throw new OSBTreeException("Error during calculation of space usage", ioe);
} finally {
atomicOperationsManager.releaseReadLock(this);
}

return new int[] { (int) (leafSpaceUsage / leafSpaceUsageCounter), (int) (nonLeafSpaceUsage / nonLeafSpaceUsageCounter) };
}

public OSBTreeCursor<K, V> iterateEntriesBetween(K keyFrom, boolean fromInclusive, K keyTo, boolean toInclusive,
boolean ascSortOrder) {
atomicOperationsManager.acquireReadLock(this);
Expand Down Expand Up @@ -1660,8 +1706,7 @@ private static enum PartialSearchMode {
/**
* Any partially matched key will be used as search result.
*/
NONE,
/**
NONE, /**
* The biggest partially matched key will be used as search result.
*/
HIGHEST_BOUNDARY,
Expand Down
Loading

0 comments on commit 09eca37

Please sign in to comment.