From 9a3173a58edd3a88e8ccf2e1c5684038c50f78dc Mon Sep 17 00:00:00 2001 From: Jingjin Zhou Date: Thu, 1 Sep 2016 11:26:12 +0800 Subject: [PATCH 1/3] [azure table storage] Add azure table storage --- azure/README.md | 73 +++++ azure/pom.xml | 46 +++ .../java/com/yahoo/ycsb/db/AzureClient.java | 279 ++++++++++++++++++ .../java/com/yahoo/ycsb/db/package-info.java | 22 ++ bin/bindings.properties | 1 + pom.xml | 2 + 6 files changed, 423 insertions(+) create mode 100644 azure/README.md create mode 100644 azure/pom.xml create mode 100644 azure/src/main/java/com/yahoo/ycsb/db/AzureClient.java create mode 100644 azure/src/main/java/com/yahoo/ycsb/db/package-info.java diff --git a/azure/README.md b/azure/README.md new file mode 100644 index 0000000000..9ddb87fb94 --- /dev/null +++ b/azure/README.md @@ -0,0 +1,73 @@ + + +## Quick Start + +This section describes how to run YCSB on Azure table storage. + +### 1. Create an Azure Storage account. +### https://azure.microsoft.com/en-us/documentation/articles/storage-create-storage-account/#create-a-storage-account + +### 2. Install Java and Maven + +### 3. Set Up YCSB + +Git clone YCSB and compile: + + git clone http://github.com/brianfrankcooper/YCSB.git + cd YCSB + mvn -pl com.yahoo.ycsb:azure-binding -am clean package + +### 4. Provide Azure Storage parameters + +Set the account name and access key. + +- `azure.account` +- `azure.key` + +Or, you can set configs with the shell command, EG: + + ./bin/ycsb load azure -s -P workloads/workloada -p azure.account=YourAccountName -p azure.key=YourAccessKey > outputLoad.txt + +### 5. Load data and run tests + +Load the data: + + ./bin/ycsb load azure -s -P workloads/workloada -p azure.account=YourAccountName -p azure.key=YourAccessKey > outputLoad.txt + +Run the workload test: + + ./bin/ycsb run azure -s -P workloads/workloada -p azure.account=YourAccountName -p azure.key=YourAccessKey > outputRun.txt + +### 6. Optional Azure Storage parameters + +- `azure.batchsize` + Could be between 1 ~ 100. Insert records to table in batch if batchsize > 1. +- `azure.protocol` + https(in default) or http. +- `azure.table` + The name of the table('usertable' in default). +- `azure.partitionkey` + The partitionkey('Test' in default). +- `azure.endpoint` + For Azure stack WOSS. + +EG: + ./bin/ycsb load azure -s -P workloads/workloada -p azure.account=YourAccountName -p azure.key=YourAccessKey -p azure.batchsize=100 -p azure.protocol=http + + + diff --git a/azure/pom.xml b/azure/pom.xml new file mode 100644 index 0000000000..fe41ea6148 --- /dev/null +++ b/azure/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + com.yahoo.ycsb + binding-parent + 0.11.0-SNAPSHOT + ../binding-parent + + + azure-binding + Azure Binding + jar + + + + com.yahoo.ycsb + core + ${project.version} + provided + + + com.microsoft.azure + azure-storage + ${azure.version} + + + diff --git a/azure/src/main/java/com/yahoo/ycsb/db/AzureClient.java b/azure/src/main/java/com/yahoo/ycsb/db/AzureClient.java new file mode 100644 index 0000000000..8df4299e41 --- /dev/null +++ b/azure/src/main/java/com/yahoo/ycsb/db/AzureClient.java @@ -0,0 +1,279 @@ +/** + * Copyright (c) 2013 - 2016 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db; + +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.table.CloudTable; +import com.microsoft.azure.storage.table.CloudTableClient; +import com.microsoft.azure.storage.table.DynamicTableEntity; +import com.microsoft.azure.storage.table.EntityProperty; +import com.microsoft.azure.storage.table.EntityResolver; +import com.microsoft.azure.storage.table.TableBatchOperation; +import com.microsoft.azure.storage.table.TableOperation; +import com.microsoft.azure.storage.table.TableQuery; +import com.microsoft.azure.storage.table.TableServiceEntity; +import com.yahoo.ycsb.ByteArrayByteIterator; +import com.yahoo.ycsb.ByteIterator; +import com.yahoo.ycsb.DB; +import com.yahoo.ycsb.DBException; +import com.yahoo.ycsb.Status; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; +import java.util.Vector; + + +/** + * YCSB binding for Azure. + * See {@code azure/README.md} for details. + */ +public class AzureClient extends DB { + + public static final String PROTOCOL = "azure.protocal"; + public static final String PROTOCOL_DEFAULT = "https"; + public static final String TABLE_ENDPOINT = "azure.endpoint"; + public static final String ACCOUNT = "azure.account"; + public static final String KEY = "azure.key"; + public static final String TABLE = "azure.table"; + public static final String TABLE_DEFAULT = "usertable"; + public static final String PARTITIONKEY = "azure.partitionkey"; + public static final String PARTITIONKEY_DEFAULT = "Test"; + public static final String BATCHSIZE = "azure.batchsize"; + public static final String BATCHSIZE_DEFAULT = "1"; + private static final int BATCHSIZE_UPPERBOUND = 100; + private static final TableBatchOperation BATCH_OPERATION = new TableBatchOperation(); + private static String partitionKey; + private CloudStorageAccount storageAccount = null; + private CloudTableClient tableClient = null; + private CloudTable cloudTable = null; + private static int batchSize; + private static int curIdx = 0; + + @Override + public void init() throws DBException { + Properties props = getProperties(); + String protocol = props.getProperty(PROTOCOL, PROTOCOL_DEFAULT); + if (protocol != "https" && protocol != "http") { + throw new DBException("Protocol must be 'http' or 'https'!\n"); + } + String table = props.getProperty(TABLE, TABLE_DEFAULT); + partitionKey = props.getProperty(PARTITIONKEY, PARTITIONKEY_DEFAULT); + batchSize = Integer.parseInt(props.getProperty(BATCHSIZE, BATCHSIZE_DEFAULT)); + if (batchSize < 1 || batchSize > BATCHSIZE_UPPERBOUND) { + throw new DBException(String.format("Batchsize must be between 1 and %d!\n", + BATCHSIZE_UPPERBOUND)); + } + String account = props.getProperty(ACCOUNT); + String key = props.getProperty(KEY); + String tableEndPoint = props.getProperty(TABLE_ENDPOINT); + String storageConnectionString = getStorageConnectionString(protocol, account, key, tableEndPoint); + try { + storageAccount = CloudStorageAccount.parse(storageConnectionString); + } catch (Exception e) { + throw new DBException("Could not connect to the account.\n", e); + } + tableClient = storageAccount.createCloudTableClient(); + try { + cloudTable = tableClient.getTableReference(table); + cloudTable.createIfNotExists(); + } catch (Exception e) { + throw new DBException("Could not connect to the table.\n", e); + } + } + + @Override + public void cleanup() { + } + + @Override + public Status read(String table, String key, Set fields, + final HashMap result) { + if (fields != null) { + return readSubset(key, fields, result); + } else { + return readEntity(key, result); + } + } + + @Override + public Status scan(String table, String startkey, int recordcount, + Set fields, Vector> result) { + try { + String whereStr = String.format("(PartitionKey eq '%s') and (RowKey ge '%s')", + partitionKey, startkey); + TableQuery scanQuery = + new TableQuery(DynamicTableEntity.class) + .where(whereStr).take(recordcount); + int cnt = 0; + for (DynamicTableEntity entity : cloudTable.execute(scanQuery)) { + HashMap properties = entity.getProperties(); + HashMap cur = new HashMap(); + for (Entry entry : properties.entrySet()) { + String fieldName = entry.getKey(); + ByteIterator fieldVal = new ByteArrayByteIterator(entry.getValue().getValueAsByteArray()); + if (fields == null || fields.contains(fieldName)) { + cur.put(fieldName, fieldVal); + } + } + result.add(cur); + if (++cnt == recordcount) { + break; + } + } + return Status.OK; + } catch (Exception e) { + return Status.ERROR; + } + } + + @Override + public Status update(String table, String key, HashMap values) { + return insertOrUpdate(key, values); + } + + @Override + public Status insert(String table, String key, HashMap values) { + if (batchSize == 1) { + return insertOrUpdate(key, values); + } else { + return insertBatch(key, values); + } + } + + @Override + public Status delete(String table, String key) { + try { + // firstly, retrieve the entity to be deleted + TableOperation retrieveOp = + TableOperation.retrieve(partitionKey, key, TableServiceEntity.class); + TableServiceEntity entity = cloudTable.execute(retrieveOp).getResultAsType(); + // secondly, delete the entity + TableOperation deleteOp = TableOperation.delete(entity); + cloudTable.execute(deleteOp); + return Status.OK; + } catch (Exception e) { + return Status.ERROR; + } + } + + private String getStorageConnectionString(String protocol, String account, String key, String tableEndPoint) { + String res = + String.format("DefaultEndpointsProtocol=%s;AccountName=%s;AccountKey=%s", + protocol, account, key); + if (tableEndPoint != null) { + res = String.format("%s;TableEndpoint=%s", res, tableEndPoint); + } + return res; + } + + /* + * Read subset of properties instead of full fields with projection. + */ + public Status readSubset(String key, Set fields, HashMap result) { + String whereStr = String.format("RowKey eq '%s'", key); + + TableQuery projectionQuery = TableQuery.from( + TableServiceEntity.class).where(whereStr).select(fields.toArray(new String[0])); + + EntityResolver> resolver = + new EntityResolver>() { + public HashMap resolve(String partitionkey, String rowKey, + Date timeStamp, HashMap properties, String etag) { + HashMap tmp = new HashMap(); + for (Entry entry : properties.entrySet()) { + String key = entry.getKey(); + ByteIterator val = new ByteArrayByteIterator(entry.getValue().getValueAsByteArray()); + tmp.put(key, val); + } + return tmp; + } + }; + try { + for (HashMap tmp : cloudTable.execute(projectionQuery, resolver)) { + for (Entry entry : tmp.entrySet()){ + String fieldName = entry.getKey(); + ByteIterator fieldVal = entry.getValue(); + result.put(fieldName, fieldVal); + } + } + return Status.OK; + } catch (Exception e) { + return Status.ERROR; + } + } + + private Status readEntity(String key, HashMap result) { + try { + // firstly, retrieve the entity to be deleted + TableOperation retrieveOp = + TableOperation.retrieve(partitionKey, key, DynamicTableEntity.class); + DynamicTableEntity entity = cloudTable.execute(retrieveOp).getResultAsType(); + HashMap properties = entity.getProperties(); + for (Entry entry: properties.entrySet()) { + String fieldName = entry.getKey(); + ByteIterator fieldVal = new ByteArrayByteIterator(entry.getValue().getValueAsByteArray()); + result.put(fieldName, fieldVal); + } + return Status.OK; + } catch (Exception e) { + return Status.ERROR; + } + } + + private Status insertBatch(String key, HashMap values) { + HashMap properties = new HashMap(); + for (Entry entry : values.entrySet()) { + String fieldName = entry.getKey(); + byte[] fieldVal = entry.getValue().toArray(); + properties.put(fieldName, new EntityProperty(fieldVal)); + } + DynamicTableEntity entity = new DynamicTableEntity(partitionKey, key, properties); + BATCH_OPERATION.insertOrReplace(entity); + if (++curIdx == batchSize) { + try { + cloudTable.execute(BATCH_OPERATION); + BATCH_OPERATION.clear(); + curIdx = 0; + } catch (Exception e) { + return Status.ERROR; + } + } + return Status.OK; + } + + private Status insertOrUpdate(String key, HashMap values) { + HashMap properties = new HashMap(); + for (Entry entry : values.entrySet()) { + String fieldName = entry.getKey(); + byte[] fieldVal = entry.getValue().toArray(); + properties.put(fieldName, new EntityProperty(fieldVal)); + } + DynamicTableEntity entity = new DynamicTableEntity(partitionKey, key, properties); + TableOperation insertOrReplace = TableOperation.insertOrReplace(entity); + try { + cloudTable.execute(insertOrReplace); + return Status.OK; + } catch (Exception e) { + return Status.ERROR; + } + } + +} diff --git a/azure/src/main/java/com/yahoo/ycsb/db/package-info.java b/azure/src/main/java/com/yahoo/ycsb/db/package-info.java new file mode 100644 index 0000000000..d1bf904127 --- /dev/null +++ b/azure/src/main/java/com/yahoo/ycsb/db/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2015 - 2016 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for Couchbase. + */ +package com.yahoo.ycsb.db; + diff --git a/bin/bindings.properties b/bin/bindings.properties index da5dac7a53..c856c08dfc 100644 --- a/bin/bindings.properties +++ b/bin/bindings.properties @@ -29,6 +29,7 @@ accumulo:com.yahoo.ycsb.db.accumulo.AccumuloClient aerospike:com.yahoo.ycsb.db.AerospikeClient asynchbase:com.yahoo.ycsb.db.AsyncHBaseClient arangodb:com.yahoo.ycsb.db.ArangoDBClient +azure:com.yahoo.ycsb.db.AzureClient basic:com.yahoo.ycsb.BasicDB cassandra-cql:com.yahoo.ycsb.db.CassandraCQLClient cassandra2-cql:com.yahoo.ycsb.db.CassandraCQLClient diff --git a/pom.xml b/pom.xml index 5357142baa..307f3419cf 100644 --- a/pom.xml +++ b/pom.xml @@ -96,6 +96,7 @@ LICENSE file. 3.1.2 5.4.0 2.7.3 + 4.0.0 @@ -107,6 +108,7 @@ LICENSE file. aerospike arangodb asynchbase + azure cassandra couchbase couchbase2 From 55c3f39013533984dbb084a44a1f3702326c72f6 Mon Sep 17 00:00:00 2001 From: Jingjin Zhou Date: Thu, 1 Sep 2016 11:34:20 +0800 Subject: [PATCH 2/3] Change version from 0.11.0 to 0.12.0 --- azure/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/pom.xml b/azure/pom.xml index fe41ea6148..d3f2346b21 100644 --- a/azure/pom.xml +++ b/azure/pom.xml @@ -22,7 +22,7 @@ LICENSE file. com.yahoo.ycsb binding-parent - 0.11.0-SNAPSHOT + 0.12.0-SNAPSHOT ../binding-parent From 9fae184fcdcc8988c8289b1efe8bc188a6999cad Mon Sep 17 00:00:00 2001 From: Jingjin Zhou Date: Wed, 28 Sep 2016 12:49:58 +0800 Subject: [PATCH 3/3] update according to feedback --- {azure => azuretablestorage}/README.md | 12 ++++++------ {azure => azuretablestorage}/pom.xml | 8 ++++---- .../ycsb/db/azuretablestorage}/AzureClient.java | 4 ++-- .../ycsb/db/azuretablestorage}/package-info.java | 6 +++--- bin/bindings.properties | 2 +- pom.xml | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) rename {azure => azuretablestorage}/README.md (68%) rename {azure => azuretablestorage}/pom.xml (87%) rename {azure/src/main/java/com/yahoo/ycsb/db => azuretablestorage/src/main/java/com/yahoo/ycsb/db/azuretablestorage}/AzureClient.java (99%) rename {azure/src/main/java/com/yahoo/ycsb/db => azuretablestorage/src/main/java/com/yahoo/ycsb/db/azuretablestorage}/package-info.java (73%) diff --git a/azure/README.md b/azuretablestorage/README.md similarity index 68% rename from azure/README.md rename to azuretablestorage/README.md index 9ddb87fb94..96c01281f5 100644 --- a/azure/README.md +++ b/azuretablestorage/README.md @@ -1,5 +1,5 @@