Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Password file system #37

Merged
merged 2 commits into from
Aug 6, 2018
Merged

Password file system #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ec94717
Read password using Beam FileSystems (allows gs://)
labianchin Aug 6, 2018
2f6eef7
Test for password file
labianchin Aug 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,13 @@ lazy val dbeamCore = project
libraryDependencies ++= Seq(
"org.slf4j" % "slf4j-simple" % slf4jVersion,
"org.apache.beam" % "beam-runners-direct-java" % beamVersion,
"org.apache.beam" % "beam-runners-google-cloud-dataflow-java" % beamVersion,
"org.postgresql" % "postgresql" % "42.2.+",
"mysql" % "mysql-connector-java" % "5.1.+",
"com.google.cloud.sql" % "postgres-socket-factory" % "1.0.5",
"com.google.cloud.sql" % "mysql-socket-factory" % "1.0.4",
"com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.9.5",
"com.google.auto.value" % "auto-value" % autoValueVersion % "provided",
"com.google.auto.value" % "auto-value" % autoValueVersion % Provided,
"org.scalatest" %% "scalatest" % "3.0.5" % "test",
"com.h2database" % "h2" % "1.4.196" % "test",
"com.typesafe.slick" %% "slick" % "3.2.0" % "test"
Expand Down
14 changes: 13 additions & 1 deletion dbeam-core/src/main/scala/com/spotify/dbeam/options/DBeamPipelineOptions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,12 @@

package com.spotify.dbeam.options

import java.nio.channels.Channels

import org.apache.beam.sdk.io.FileSystems
import org.apache.beam.sdk.options.Validation.Required
import org.apache.beam.sdk.options.{Default, Description, PipelineOptions}
import org.slf4j.{Logger, LoggerFactory}

import scala.io.Source

Expand Down Expand Up @@ -114,8 +118,16 @@ trait OutputOptions extends PipelineOptions {
}

object PipelineOptionsUtil {
val log: Logger = LoggerFactory.getLogger(PipelineOptionsUtil.getClass)
def readPassword(options: DBeamPipelineOptions): Option[String] = {
Option(options.getPasswordFile).map(Source.fromFile(_).mkString.stripLineEnd)
FileSystems.setDefaultPipelineOptions(options)
Option(options.getPasswordFile)
.map(FileSystems.matchSingleFileSpec)
.map{m =>
log.info("Reading password from file: {}", m.resourceId().toString)
Channels.newInputStream(FileSystems.open(m.resourceId()))
}
.map(Source.fromInputStream(_).mkString.stripLineEnd)
.orElse(Option(options.getPassword))
}
}
5 changes: 4 additions & 1 deletion dbeam-core/src/test/scala/com/spotify/dbeam/JdbcAvroJobTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,19 @@ class JdbcAvroJobTest extends FlatSpec with Matchers with BeforeAndAfterAll {
"jdbc:h2:mem:test2;MODE=PostgreSQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1"
private val db: Database = Database.forURL(connectionUrl, driver = "org.h2.Driver")
private val dir = tmpDir
private val passwordFile = new File(dir.getAbsolutePath + ".password")

def tmpDir: File = new File(
new File(sys.props("java.io.tmpdir")),
"jdbc-avro-test-" + UUID.randomUUID().toString)

override def beforeAll(): Unit = {
JdbcTestFixtures.createFixtures(db, Seq(JdbcTestFixtures.record1, JdbcTestFixtures.record2))
passwordFile.createNewFile()
}

override protected def afterAll(): Unit = {
passwordFile.delete()
Files.walk(dir.toPath)
.sorted(Comparator.reverseOrder())
.iterator()
Expand All @@ -60,7 +63,7 @@ class JdbcAvroJobTest extends FlatSpec with Matchers with BeforeAndAfterAll {
"--skipPartitionCheck",
"--connectionUrl=" + connectionUrl,
"--username=",
"--password=",
"--passwordFile=" + passwordFile.getAbsolutePath,
"--table=coffees",
"--output=" + dir.getAbsolutePath)
)
Expand Down