Skip to content

Commit

Permalink
Merge pull request #9 from civitaspo/use-embulk-test-instead-of-Embul…
Browse files Browse the repository at this point in the history 
…kEmbed

Use TestingEmbulk instead of EmbulkEmbed
  • Loading branch information
@higarin
higarin authored Jan 26, 2020
2 parents 06fbdb4 + 57d951c commit 4f9b868
Show file tree
Hide file tree
Showing 5 changed files with 127 additions and 203 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ dependencies {
testCompile "org.embulk:embulk-standards:0.9.23"
testCompile "org.embulk:embulk-deps-buffer:0.9.23"
testCompile "org.embulk:embulk-deps-config:0.9.23"
testCompile "org.embulk:embulk-core:0.9.23:tests"
testCompile "org.embulk:embulk-test:0.9.23"
}

compileScala {
Expand Down
91 changes: 27 additions & 64 deletions src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala
View file
Original file line number Diff line number Diff line change
@@ -1,135 +1,98 @@
package org.embulk.input.dynamodb

import java.io.File
import java.nio.charset.Charset
import java.nio.file.{FileSystems, Files}
import java.nio.file.Files

import com.google.inject.{Binder, Module}
import org.embulk.config.ConfigSource
import org.embulk.exec.PartialExecutionException
import org.embulk.plugin.InjectedPluginSource
import org.embulk.spi.InputPlugin
import org.embulk.{EmbulkEmbed, EmbulkTestRuntime}
import org.embulk.input.dynamodb.testutil.EmbulkTestBase
import org.junit.Assert.assertEquals
import org.junit.{Before, Rule, Test}

class AwsCredentialsTest {
private var EMBULK_DYNAMODB_TEST_REGION: String = null
private var EMBULK_DYNAMODB_TEST_TABLE: String = null
private var EMBULK_DYNAMODB_TEST_ACCESS_KEY: String = null
private var EMBULK_DYNAMODB_TEST_SECRET_KEY: String = null
private var EMBULK_DYNAMODB_TEST_PROFILE_NAME: String = null

private var embulk: EmbulkEmbed = null

@Rule
def runtime: EmbulkTestRuntime = new EmbulkTestRuntime

@Before
def createResources(): Unit = {
// Get Environments
EMBULK_DYNAMODB_TEST_REGION = System.getenv("EMBULK_DYNAMODB_TEST_REGION")
EMBULK_DYNAMODB_TEST_TABLE = System.getenv("EMBULK_DYNAMODB_TEST_TABLE")
EMBULK_DYNAMODB_TEST_ACCESS_KEY = System.getenv("EMBULK_DYNAMODB_TEST_ACCESS_KEY")
EMBULK_DYNAMODB_TEST_SECRET_KEY = System.getenv("EMBULK_DYNAMODB_TEST_SECRET_KEY")
EMBULK_DYNAMODB_TEST_PROFILE_NAME = System.getenv("EMBULK_DYNAMODB_TEST_PROFILE_NAME")

val bootstrap = new EmbulkEmbed.Bootstrap()
bootstrap.addModules(new Module {
def configure(binder: Binder): Unit = {
InjectedPluginSource.registerPluginTo(binder,
classOf[InputPlugin],
"dynamodb",
classOf[DynamodbInputPlugin])
}
})

embulk = bootstrap.initialize()
}
import org.junit.Test

class AwsCredentialsTest extends EmbulkTestBase {
private val EMBULK_DYNAMODB_TEST_REGION = System.getenv("EMBULK_DYNAMODB_TEST_REGION")
private val EMBULK_DYNAMODB_TEST_TABLE = System.getenv("EMBULK_DYNAMODB_TEST_TABLE")
private val EMBULK_DYNAMODB_TEST_ACCESS_KEY = System.getenv("EMBULK_DYNAMODB_TEST_ACCESS_KEY")
private val EMBULK_DYNAMODB_TEST_SECRET_KEY = System.getenv("EMBULK_DYNAMODB_TEST_SECRET_KEY")
private val EMBULK_DYNAMODB_TEST_PROFILE_NAME = System.getenv("EMBULK_DYNAMODB_TEST_PROFILE_NAME")

def doTest(config: ConfigSource): Unit = {
embulk.run(config)
def doTest(inConfig: ConfigSource): Unit = {
val path = embulk.createTempFile("csv")
embulk.runInput(inConfig, path)

val fs = FileSystems.getDefault
val lines = Files.readAllLines(fs.getPath("result000.00.tsv"), Charset.forName("UTF-8"))
assertEquals("KEY-1\t1\tHogeHoge", lines.get(0))
val lines = Files.readAllLines(path, Charset.forName("UTF-8"))
assertEquals("KEY-1,1,HogeHoge", lines.get(0))
}

@Test
def notSetAuthMethod_SetCredentials(): Unit = {
val config = embulk.newConfigLoader().fromYamlFile(
new File("src/test/resources/yaml/notSetAuthMethod.yml"))
val config = embulk.loadYamlResource("yaml/notSetAuthMethod.yml")

config.getNested("in")
.set("region", EMBULK_DYNAMODB_TEST_REGION)
.set("table", EMBULK_DYNAMODB_TEST_TABLE)
.set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
.set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)

doTest(config)
doTest(config.getNested("in"))
}

@Test
def setAuthMethod_Basic(): Unit = {
val config = embulk.newConfigLoader().fromYamlFile(
new File("src/test/resources/yaml/authMethodBasic.yml"))
val config = embulk.loadYamlResource("yaml/authMethodBasic.yml")

config.getNested("in")
.set("region", EMBULK_DYNAMODB_TEST_REGION)
.set("table", EMBULK_DYNAMODB_TEST_TABLE)
.set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
.set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)

doTest(config)
doTest(config.getNested("in"))
}

@Test(expected = classOf[PartialExecutionException])
def setAuthMethod_Basic_NotSet(): Unit = {
val config = embulk.newConfigLoader().fromYamlFile(
new File("src/test/resources/yaml/authMethodBasic_Error.yml"))
val config = embulk.loadYamlResource("yaml/authMethodBasic_Error.yml")

config.getNested("in")
.set("region", EMBULK_DYNAMODB_TEST_REGION)
.set("table", EMBULK_DYNAMODB_TEST_TABLE)

doTest(config)
doTest(config.getNested("in"))
}

@Test
def setAuthMethod_Env(): Unit = {
val config = embulk.newConfigLoader().fromYamlFile(
new File("src/test/resources/yaml/authMethodEnv.yml"))
val config = embulk.loadYamlResource("yaml/authMethodEnv.yml")

config.getNested("in")
.set("region", EMBULK_DYNAMODB_TEST_REGION)
.set("table", EMBULK_DYNAMODB_TEST_TABLE)

doTest(config)
doTest(config.getNested("in"))
}

@Test
def setAuthMethod_Profile(): Unit = {
val config = embulk.newConfigLoader().fromYamlFile(
new File("src/test/resources/yaml/authMethodProfile.yml"))
val config = embulk.loadYamlResource("yaml/authMethodProfile.yml")

config.getNested("in")
.set("region", EMBULK_DYNAMODB_TEST_REGION)
.set("table", EMBULK_DYNAMODB_TEST_TABLE)
.set("profile_name", EMBULK_DYNAMODB_TEST_PROFILE_NAME)

doTest(config)
doTest(config.getNested("in"))
}

@Test(expected = classOf[PartialExecutionException])
def setAuthMethod_Profile_NotExistProfileName(): Unit = {
val config = embulk.newConfigLoader().fromYamlFile(
new File("src/test/resources/yaml/authMethodProfile.yml"))
val config = embulk.loadYamlResource("yaml/authMethodProfile.yml")

config.getNested("in")
.set("region", EMBULK_DYNAMODB_TEST_REGION)
.set("table", EMBULK_DYNAMODB_TEST_TABLE)
.set("profile_name", "NotExistName")

doTest(config)
doTest(config.getNested("in"))
}
}
106 changes: 38 additions & 68 deletions src/test/scala/org/embulk/input/dynamodb/ope/QueryOperationTest.scala
View file
Original file line number Diff line number Diff line change
@@ -1,84 +1,54 @@
package org.embulk.input.dynamodb.ope

import java.io.File
import java.nio.charset.Charset
import java.nio.file.{FileSystems, Files}

import com.fasterxml.jackson.databind.ObjectMapper
import com.google.inject.{Binder, Module}
import org.embulk.EmbulkEmbed
import org.embulk.config.ConfigSource
import org.embulk.input.dynamodb.DynamodbInputPlugin
import org.embulk.plugin.InjectedPluginSource
import org.embulk.spi.InputPlugin
import org.embulk.input.dynamodb.testutil.EmbulkTestBase
import org.embulk.spi.util.Pages
import org.hamcrest.CoreMatchers._
import org.hamcrest.MatcherAssert.assertThat
import org.junit.Assert._
import org.junit.{Before, Test}

class QueryOperationTest {
private var embulk: EmbulkEmbed = null

private var EMBULK_DYNAMODB_TEST_TABLE: String = null
private var mapper: ObjectMapper = null

@Before
def createResources(): Unit = {
// Get Environments
EMBULK_DYNAMODB_TEST_TABLE = System.getenv("EMBULK_DYNAMODB_TEST_TABLE")

val bootstrap = new EmbulkEmbed.Bootstrap()
bootstrap.addModules(new Module {
def configure(binder: Binder): Unit = {
InjectedPluginSource.registerPluginTo(binder,
classOf[InputPlugin],
"dynamodb",
classOf[DynamodbInputPlugin])
}
})

embulk = bootstrap.initialize()

mapper = new ObjectMapper()
}


def doTest(config: ConfigSource): Unit = {
embulk.run(config)

val fs = FileSystems.getDefault
val lines = Files.readAllLines(fs.getPath("dynamodb-local-result000.00.tsv"), Charset.forName("UTF-8"))
assertEquals(lines.size, 1)

val head = lines.get(0)
val values = head.split("\t")

assertThat(values(0), is("key-1"))
assertThat(values(1), is("0"))
assertThat(values(2), is("42.195"))
assertThat(values(3), is("true"))

val listValue = mapper.readValue(values(4).replaceAll("\"(?!\")", ""), classOf[java.util.List[Object]])
assertThat(listValue.size(), is(2))
assertThat(listValue.get(0).asInstanceOf[String], is("list-value"))
assertThat(listValue.get(1).asInstanceOf[Int], is(123))

val mapValue = mapper.readValue(values(5).replaceAll("\"(?!\")", ""), classOf[java.util.Map[String, Object]])
assert(mapValue.containsKey("map-key-1"))
assertThat(mapValue.get("map-key-1").asInstanceOf[String], is("map-value-1"))
assert(mapValue.containsKey("map-key-2"))
assertThat(mapValue.get("map-key-2").asInstanceOf[Int], is(456))
import org.junit.Test
import org.msgpack.value.Value

import scala.jdk.CollectionConverters._

class QueryOperationTest extends EmbulkTestBase {
private val EMBULK_DYNAMODB_TEST_TABLE: String = System.getenv("EMBULK_DYNAMODB_TEST_TABLE")

def doTest(inConfig: ConfigSource): Unit = {
val path = embulk.createTempFile("csv")
val result = embulk
.inputBuilder()
.in(inConfig)
.outputPath(path)
.preview()

val pages = result.getPages
val head = Pages.toObjects(result.getSchema, pages.get(0)).get(0)

assertThat(head(0).toString, is("key-1"))
assertThat(head(1).asInstanceOf[Long], is(0L))
assertThat(head(2).asInstanceOf[Double], is(42.195))
assertThat(head(3).asInstanceOf[Boolean], is(true))

val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
assertThat(arrayValue.size(), is(2))
assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))

val mapValue = head(5).asInstanceOf[Value].asMapValue()
assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
assertThat(mapValue.entrySet().asScala.filter(_.getKey.toString.equals("map-key-1")).head.getValue.toString, is("map-value-1"))
assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
assertThat(mapValue.entrySet().asScala.filter(_.getKey.toString.equals("map-key-2")).head.getValue.asIntegerValue().asLong(), is(456L))
}

@Test
def queryTest(): Unit = {
val config = embulk.newConfigLoader().fromYamlFile(
new File("src/test/resources/yaml/dynamodb-local-query.yml"))
val config = embulk.loadYamlResource("yaml/dynamodb-local-query.yml")

config.getNested("in")
.set("operation", "query")
.set("table", EMBULK_DYNAMODB_TEST_TABLE)

doTest(config)
doTest(config.getNested("in"))
}
}
Loading

0 comments on commit 4f9b868

Please sign in to comment.