Skip to content

Commit

Permalink
[MINOR][SQL] Use resource path for test_script.sh
Browse files Browse the repository at this point in the history
## What changes were proposed in this pull request?
This PR modified the test case `test("script")` to use resource path for `test_script.sh`. Make the test case portable (even in IntelliJ).

## How was this patch tested?
Passed the test case.
Before:
Run `test("script")` in IntelliJ:
```
Caused by: org.apache.spark.SparkException: Subprocess exited with status 127. Error: bash: src/test/resources/test_script.sh: No such file or directory
```
After:
Test passed.

Author: Weiqing Yang <yangweiqing001@gmail.com>

Closes apache#15246 from weiqingy/hivetest.
  • Loading branch information
weiqingy authored and uzadude committed Jan 27, 2017
1 parent 5ed029d commit 78aad64
Show file tree
Hide file tree
Showing 6 changed files with 28 additions and 12 deletions.
11 changes: 11 additions & 0 deletions core/src/test/scala/org/apache/spark/SparkFunSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
package org.apache.spark

// scalastyle:off
import java.io.File

import org.scalatest.{BeforeAndAfterAll, FunSuite, Outcome}

import org.apache.spark.internal.Logging
Expand All @@ -41,6 +43,15 @@ abstract class SparkFunSuite
}
}

// helper function
protected final def getTestResourceFile(file: String): File = {
new File(getClass.getClassLoader.getResource(file).getFile)
}

protected final def getTestResourcePath(file: String): String = {
getTestResourceFile(file).getCanonicalPath
}

/**
* Log the suite name and the test name before and after each test.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,16 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
with JsonTestUtils with Eventually with WebBrowser with LocalSparkContext
with ResetSystemProperties {

private val logDir = new File("src/test/resources/spark-events")
private val expRoot = new File("src/test/resources/HistoryServerExpectations/")
private val logDir = getTestResourcePath("spark-events")
private val expRoot = getTestResourceFile("HistoryServerExpectations")

private var provider: FsHistoryProvider = null
private var server: HistoryServer = null
private var port: Int = -1

def init(): Unit = {
val conf = new SparkConf()
.set("spark.history.fs.logDirectory", logDir.getAbsolutePath)
.set("spark.history.fs.logDirectory", logDir)
.set("spark.history.fs.update.interval", "0")
.set("spark.testing", "true")
provider = new FsHistoryProvider(conf)
Expand Down
3 changes: 2 additions & 1 deletion core/src/test/scala/org/apache/spark/ui/UISuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,10 @@ class UISuite extends SparkFunSuite {
}

private def sslEnabledConf(): (SparkConf, SSLOptions) = {
val keyStoreFilePath = getTestResourcePath("spark.keystore")
val conf = new SparkConf()
.set("spark.ssl.ui.enabled", "true")
.set("spark.ssl.ui.keyStore", "./src/test/resources/spark.keystore")
.set("spark.ssl.ui.keyStore", keyStoreFilePath)
.set("spark.ssl.ui.keyStorePassword", "123456")
.set("spark.ssl.ui.keyPassword", "123456")
(conf, new SecurityManager(conf).getSSLOptions("ui"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ class LogicalPlanToSQLSuite extends SQLBuilderTest with SQLTestUtils {

// Used for generating new query answer files by saving
private val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
private val goldenSQLPath = "src/test/resources/sqlgen/"
private val goldenSQLPath = getTestResourcePath("sqlgen")

protected override def beforeAll(): Unit = {
super.beforeAll()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ class HiveSparkSubmitSuite
case v if v.startsWith("2.10") || v.startsWith("2.11") => v.substring(0, 4)
case x => throw new Exception(s"Unsupported Scala Version: $x")
}
val testJar = s"sql/hive/src/test/resources/regression-test-SPARK-8489/test-$version.jar"
val jarDir = getTestResourcePath("regression-test-SPARK-8489")
val testJar = s"$jarDir/test-$version.jar"
val args = Seq(
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,14 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
import spark.implicits._

test("script") {
val scriptFilePath = getTestResourcePath("test_script.sh")
if (testCommandAvailable("bash") && testCommandAvailable("echo | sed")) {
val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
df.createOrReplaceTempView("script_table")
val query1 = sql(
"""
s"""
|SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
|REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS
|REDUCE c1, c2, c3 USING 'bash $scriptFilePath' AS
|(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
}
Expand Down Expand Up @@ -1290,11 +1291,12 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
.selectExpr("id AS a", "id AS b")
.createOrReplaceTempView("test")

val scriptFilePath = getTestResourcePath("data")
checkAnswer(
sql(
"""FROM(
s"""FROM(
| FROM test SELECT TRANSFORM(a, b)
| USING 'python src/test/resources/data/scripts/test_transform.py "\t"'
| USING 'python $scriptFilePath/scripts/test_transform.py "\t"'
| AS (c STRING, d STRING)
|) t
|SELECT c
Expand All @@ -1308,12 +1310,13 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
.selectExpr("id AS a", "id AS b")
.createOrReplaceTempView("test")

val scriptFilePath = getTestResourcePath("data")
val df = sql(
"""FROM test
s"""FROM test
|SELECT TRANSFORM(a, b)
|ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
|WITH SERDEPROPERTIES('field.delim' = '|')
|USING 'python src/test/resources/data/scripts/test_transform.py "|"'
|USING 'python $scriptFilePath/scripts/test_transform.py "|"'
|AS (c STRING, d STRING)
|ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
|WITH SERDEPROPERTIES('field.delim' = '|')
Expand Down

0 comments on commit 78aad64

Please sign in to comment.