From 8cf399a5f066f6736dd3b808657f354ece2e7718 Mon Sep 17 00:00:00 2001 From: foryou2030 Date: Tue, 22 Nov 2016 14:13:29 +0800 Subject: [PATCH 1/2] fix query error --- .../client/DictionaryClientHandler.java | 1 - .../IncrementalColumnDictionaryGenerator.java | 51 ++++++++---- .../carbondata/examples/CarbonExample.scala | 24 +++--- .../examples/CarbonExampleForDebug.scala | 82 +++++++++++++++++++ 4 files changed, 128 insertions(+), 30 deletions(-) create mode 100644 examples/src/main/scala/org/apache/carbondata/examples/CarbonExampleForDebug.scala diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java index 78d1f103aba..3f919900e58 100644 --- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java +++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java @@ -43,7 +43,6 @@ public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) thr @Override public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { DictionaryKey key = (DictionaryKey) e.getMessage(); - System.out.println(key.getData()); dictKeyQueue.offer(key); super.messageReceived(ctx, e); } diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java index 905ea95e03e..3559095fc67 100644 --- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java @@ -69,6 +69,8 @@ public class IncrementalColumnDictionaryGenerator private CarbonDimension dimension; + private CarbonDictionaryWriter dictionaryWriter = null; + public IncrementalColumnDictionaryGenerator(CarbonDimension dimension, int maxValue) { this.maxDictionary = maxValue; this.dimension = dimension; @@ -100,6 +102,7 @@ public IncrementalColumnDictionaryGenerator(CarbonDimension dimension, int maxVa if (dict == null) { dict = ++maxDictionary; incrementalCache.put(value, dict); + reverseIncrementalCache.put(dict, value); } return dict; } @@ -137,6 +140,8 @@ public IncrementalColumnDictionaryGenerator(CarbonDimension dimension, int maxVa // write sort index writeSortIndex(distinctValues, dictionary, dictionaryService, tableIdentifier, columnIdentifier, storePath); + // update Meta Data + updateMetaData(); } /** @@ -175,7 +180,7 @@ private List writeDictionary(Dictionary dictionary, String storePath, CarbonTablePath carbonTablePath) throws IOException { List distinctValues = new ArrayList<>(); - CarbonDictionaryWriter dictionaryWriter = null; +// CarbonDictionaryWriter dictionaryWriter = null; try { dictionaryWriter = dictionaryService .getDictionaryWriter(dictIdentifier.getCarbonTableIdentifier(), @@ -184,24 +189,27 @@ private List writeDictionary(Dictionary dictionary, dictionaryWriter.write(CarbonCommonConstants.MEMBER_DEFAULT_VAL); distinctValues.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); } - if (incrementalCache.size() > 0) { - // TODO: map need sort first - for (String key : incrementalCache.keySet()) { - reverseIncrementalCache.put(incrementalCache.get(key), key); - } - for (int i = 2; i < reverseIncrementalCache.size() + 2; i++) { - String parsedValue = DataTypeUtil - .normalizeColumnValueForItsDataType(reverseIncrementalCache.get(i), dimension); - if (null != parsedValue) { - if (!isDictionaryExists(dictIdentifier, carbonTablePath)) { + // write value to dictionary file + if (reverseIncrementalCache.size() >= 1) { + if (isDictionaryExists(dictIdentifier, carbonTablePath)) { + for (int i = 2; i < reverseIncrementalCache.size() + 2; i++) { + String value = reverseIncrementalCache.get(i); + String parsedValue = DataTypeUtil + .normalizeColumnValueForItsDataType(value, dimension); + if (null != parsedValue && dictionary.getSurrogateKey(parsedValue) == + CarbonCommonConstants.INVALID_SURROGATE_KEY) { + dictionaryWriter.write(parsedValue); + distinctValues.add(parsedValue); + } + } + } else { + for (int i = 2; i < reverseIncrementalCache.size() + 2; i++) { + String value = reverseIncrementalCache.get(i); + String parsedValue = DataTypeUtil + .normalizeColumnValueForItsDataType(value, dimension); + if (null != parsedValue) { dictionaryWriter.write(parsedValue); distinctValues.add(parsedValue); - } else { - if (dictionary.getSurrogateKey(parsedValue) == - CarbonCommonConstants.INVALID_SURROGATE_KEY) { - dictionaryWriter.write(parsedValue); - distinctValues.add(parsedValue); - } } } } @@ -256,4 +264,13 @@ private void writeSortIndex(List distinctValues, } } } + + /** + * update dictionary metadata + */ + private void updateMetaData() throws IOException{ + if (null != dictionaryWriter) { + dictionaryWriter.commit(); + } + } } diff --git a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala index a404bac8e7a..f98d46d6104 100644 --- a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala +++ b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala @@ -45,22 +45,22 @@ object CarbonExample { // in each node to do data loading, another uses a multi-thread framework without Kettle (See // AbstractDataLoadProcessorStep) // Load data with Kettle -// cc.sql(s""" -// LOAD DATA LOCAL INPATH '$testData' into table t3 -// """) -// -// // Perform a query -// cc.sql(""" -// SELECT country, count(salary) AS amount -// FROM t3 -// WHERE country IN ('china','france') -// GROUP BY country -// """).show() + cc.sql(s""" + LOAD DATA LOCAL INPATH '$testData' into table t3 + """) + + // Perform a query + cc.sql(""" + SELECT country, count(salary) AS amount + FROM t3 + WHERE country IN ('china','france') + GROUP BY country + """).show() // Load data without kettle cc.sql(s""" LOAD DATA LOCAL INPATH '$testData' into table t3 - OPTIONS('USE_KETTLE'='false', 'USEONEPASS'='true') + OPTIONS('USE_KETTLE'='false') """) // Perform a query diff --git a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExampleForDebug.scala b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExampleForDebug.scala new file mode 100644 index 00000000000..b7292ef7efc --- /dev/null +++ b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExampleForDebug.scala @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.examples.util.ExampleUtils + +object CarbonExampleForDebug { + + def main(args: Array[String]) { + val cc = ExampleUtils.createCarbonContext("CarbonExample") + val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv" + + // Specify timestamp format based on raw data + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd") + +// cc.sql("DROP TABLE IF EXISTS t3_origin") +// +// // Create table, 6 dimensions, 1 measure +// cc.sql(""" +// CREATE TABLE IF NOT EXISTS t3_origin +// (ID Int, date Timestamp, country String, +// name String, phonetype String, serialname String, salary Int) +// STORED BY 'carbondata' +// """) +// +// // Load data without kettle +// cc.sql(s""" +// LOAD DATA LOCAL INPATH '$testData' into table t3_origin +// OPTIONS('USE_KETTLE'='false') +// """) +// +// cc.sql(""" +// SELECT count(distinct country), count(distinct name), count(distinct phonetype), count(distinct serialname) FROM t3_origin +// """).show() + + cc.sql("DROP TABLE IF EXISTS t3") + + // Create table, 6 dimensions, 1 measure + cc.sql(""" + CREATE TABLE IF NOT EXISTS t3 + (ID Int, date Timestamp, country String, + name String, phonetype String, serialname String, salary Int) + STORED BY 'carbondata' + """) + + // Load data without kettle + cc.sql(s""" + LOAD DATA LOCAL INPATH '$testData' into table t3 + OPTIONS('USE_KETTLE'='false', 'USEONEPASS'='true') + """) + + // Perform a query + cc.sql(""" + SELECT country, count(salary) AS amount + FROM t3 + WHERE country IN ('china','france') + GROUP BY country + """).show() + + // Drop table +// cc.sql("DROP TABLE IF EXISTS t3") + } + +} From e5e657f44893ba2251569f6836110b5026fd2741 Mon Sep 17 00:00:00 2001 From: foryou2030 Date: Tue, 22 Nov 2016 14:15:48 +0800 Subject: [PATCH 2/2] remove no used files --- .../examples/CarbonExampleForDebug.scala | 82 ------------------- 1 file changed, 82 deletions(-) delete mode 100644 examples/src/main/scala/org/apache/carbondata/examples/CarbonExampleForDebug.scala diff --git a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExampleForDebug.scala b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExampleForDebug.scala deleted file mode 100644 index b7292ef7efc..00000000000 --- a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExampleForDebug.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.examples - -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.util.CarbonProperties -import org.apache.carbondata.examples.util.ExampleUtils - -object CarbonExampleForDebug { - - def main(args: Array[String]) { - val cc = ExampleUtils.createCarbonContext("CarbonExample") - val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv" - - // Specify timestamp format based on raw data - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd") - -// cc.sql("DROP TABLE IF EXISTS t3_origin") -// -// // Create table, 6 dimensions, 1 measure -// cc.sql(""" -// CREATE TABLE IF NOT EXISTS t3_origin -// (ID Int, date Timestamp, country String, -// name String, phonetype String, serialname String, salary Int) -// STORED BY 'carbondata' -// """) -// -// // Load data without kettle -// cc.sql(s""" -// LOAD DATA LOCAL INPATH '$testData' into table t3_origin -// OPTIONS('USE_KETTLE'='false') -// """) -// -// cc.sql(""" -// SELECT count(distinct country), count(distinct name), count(distinct phonetype), count(distinct serialname) FROM t3_origin -// """).show() - - cc.sql("DROP TABLE IF EXISTS t3") - - // Create table, 6 dimensions, 1 measure - cc.sql(""" - CREATE TABLE IF NOT EXISTS t3 - (ID Int, date Timestamp, country String, - name String, phonetype String, serialname String, salary Int) - STORED BY 'carbondata' - """) - - // Load data without kettle - cc.sql(s""" - LOAD DATA LOCAL INPATH '$testData' into table t3 - OPTIONS('USE_KETTLE'='false', 'USEONEPASS'='true') - """) - - // Perform a query - cc.sql(""" - SELECT country, count(salary) AS amount - FROM t3 - WHERE country IN ('china','france') - GROUP BY country - """).show() - - // Drop table -// cc.sql("DROP TABLE IF EXISTS t3") - } - -}