Skip to content

Commit

Permalink
Fixed Performance issue for dictionary loading during decoder
Browse files Browse the repository at this point in the history
  • Loading branch information
kumarvishal09 committed Jan 8, 2017
1 parent 96ef7f4 commit 3fa0b0f
Showing 1 changed file with 30 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.spark.sql

import scala.collection.JavaConverters._

import org.apache.spark.TaskContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
Expand Down Expand Up @@ -224,21 +226,39 @@ case class CarbonDictionaryDecoder(
}

private def getDictionary(atiMap: Map[String, AbsoluteTableIdentifier],
cache: Cache[DictionaryColumnUniqueIdentifier, Dictionary]) = {
val dicts: Seq[Dictionary] = getDictionaryColumnIds.map { f =>
cache: Cache[DictionaryColumnUniqueIdentifier, Dictionary]) = {
val dictionaryColumnIds = getDictionaryColumnIds.map { f =>
if (f._2 != null) {
try {
cache.get(new DictionaryColumnUniqueIdentifier(
atiMap(f._1).getCarbonTableIdentifier,
f._2, f._3))
} catch {
case _: Throwable => null
}
new DictionaryColumnUniqueIdentifier(
atiMap(f._1).getCarbonTableIdentifier,
f._2, f._3)
} else {
null
}
}
dicts
try {
val noDictionaryIndexes = new java.util.ArrayList[Int]()
dictionaryColumnIds.zipWithIndex.foreach { x =>
if (x._1 == null) {
noDictionaryIndexes.add(x._2)
}
}
val dict = cache.getAll(dictionaryColumnIds.filter(_ != null).toSeq.asJava);
val finalDict = new java.util.ArrayList[Dictionary]()
var dictIndex: Int = 0
dictionaryColumnIds.zipWithIndex.foreach { x =>
if (!noDictionaryIndexes.contains(x._2)) {
finalDict.add(dict.get(dictIndex))
dictIndex += 1
} else {
finalDict.add(null)
}
}
finalDict.asScala
} catch {
case t: Throwable => Seq.empty
}

}

}

0 comments on commit 3fa0b0f

Please sign in to comment.