Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CARBONDATA-46]Fixed dataframe api in carbon #31

Closed
wants to merge 2 commits into from
Closed

[CARBONDATA-46]Fixed dataframe api in carbon #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
94b192d
Fixed dataframe api
ravipesala Jul 8, 2016
fccae0d
Fixed relative path issue of metastore location
ravipesala Jul 9, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,12 @@ class CarbonContext(
override protected def configure(): Map[String, String] = {
sc.hadoopConfiguration.addResource("hive-site.xml")
if (sc.hadoopConfiguration.get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
val hiveMetaStoreDB = metaStorePath + "/metastore_db"
val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
logDebug(s"metastore db is going to be created in location : $hiveMetaStoreDB")
super.configure() ++ Map((CarbonCommonConstants.HIVE_CONNECTION_URL,
s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"),
("hive.metastore.warehouse.dir", metaStorePath + "/hivemetadata"))
("hive.metastore.warehouse.dir", metaStorePathAbsolute + "/hivemetadata"))
} else {
super.configure()
}
Expand Down
27 changes: 13 additions & 14 deletions integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,19 @@ class CarbonSource
override def createRelation(
sqlContext: SQLContext,
parameters: Map[String, String]): BaseRelation = {
if (parameters.get("tablePath") != None) {
val options = new CarbonOption(parameters)
val tableIdentifier = options.tableIdentifier.split("""\.""").toSeq
val ident = tableIdentifier match {
case Seq(name) => TableIdentifier(name, None)
case Seq(db, name) => TableIdentifier(name, Some(db))
}
CarbonDatasourceRelation(ident, None)(sqlContext)
} else if (parameters.get("path") != None) {
CarbonDatasourceHadoopRelation(sqlContext, Array(parameters.get("path").get), parameters)
} else {
sys.error("Carbon table path not found")
// if path is provided we can directly create Hadoop relation. \
// Otherwise create datasource relation
parameters.get("path") match {
case Some(path) => CarbonDatasourceHadoopRelation(sqlContext, Array(path), parameters)
case _ =>
val options = new CarbonOption(parameters)
val tableIdentifier = options.tableIdentifier.split("""\.""").toSeq
val identifier = tableIdentifier match {
case Seq(name) => TableIdentifier(name, None)
case Seq(db, name) => TableIdentifier(name, Some(db))
}
CarbonDatasourceRelation(identifier, None)(sqlContext)
}

}

override def createRelation(
Expand Down Expand Up @@ -93,7 +92,7 @@ class CarbonSource
sys.error(s"ErrorIfExists mode, path $storePath already exists.")
case (SaveMode.Overwrite, true) =>
val cc = CarbonContext.getInstance(sqlContext.sparkContext)
cc.sql(s"DROP CUBE IF EXISTS ${ options.dbName }.${ options.tableName }")
cc.sql(s"DROP TABLE IF EXISTS ${ options.dbName }.${ options.tableName }")
(true, false)
case (SaveMode.Overwrite, false) | (SaveMode.ErrorIfExists, false) =>
(true, false)
Expand Down
2 changes: 1 addition & 1 deletion integration/spark/src/main/scala/org/carbondata/spark/package.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ package object spark {
if (f.getPath.getName.startsWith("part-")) {
val newPath = s"${ f.getPath.getParent }/${ f.getPath.getName }.csv"
if (!fs.rename(f.getPath, new Path(newPath))) {
cc.sql(s"DROP CUBE ${ options.tableName }")
cc.sql(s"DROP TABLE ${ options.tableName }")
throw new RuntimeException("File system rename failed when loading data into carbon")
}
}
Expand Down