Add support for Avro records (#5)

* Add publish configuration for Maven Central releases * Allow Scala Tests action to be manually triggered (#2) Signed-off-by: Dr. Daniel Alexander Smith <daniel@danielsmith.eu> * add avro support for RecordValue object (#1) * add avro support for RecordValue object Signed-off-by: Dr. Daniel Alexander Smith <daniel@danielsmith.eu> * use avro version that doesnt conflict with jackson --------- Signed-off-by: Dr. Daniel Alexander Smith <daniel@danielsmith.eu> Co-authored-by: Dr. Daniel Alexander Smith <daniel@danielsmith.eu> * Bump to v1.1.0 Signed-off-by: Dr. Daniel Alexander Smith <daniel@danielsmith.eu> * Create CODEOWNERS Signed-off-by: Dr. Daniel Alexander Smith <daniel@danielsmith.eu> --------- Signed-off-by: Dr. Daniel Alexander Smith <daniel@danielsmith.eu> Co-authored-by: Edward-Jones-6point6 <88337463+Edward-Jones-6point6@users.noreply.github.com>
Accenture · Sep 1, 2023 · d9104b6 · d9104b6
1 parent c1284a6
commit d9104b6
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 13 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @danielsmith-eu
diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml
@@ -6,6 +6,7 @@
 name: Scala CI - Run Tests
 
 on:
+  workflow_dispatch:
   push:
     branches: [ "main" ]
   pull_request:

diff --git a/data-quality-profiler/build.sbt b/data-quality-profiler/build.sbt
@@ -1,7 +1,7 @@
-ThisBuild / organization := "uk.gov.ipt.das"
-ThisBuild / version      := "1.0.0"
+ThisBuild / organization := "io.github.6point6"
+ThisBuild / version      := "1.1.0"
 
-name := "das-data-profiler"
+name := "data-quality-profiler-and-rules-engine"
 
 scalaVersion := "2.12.15"
 
@@ -63,6 +63,9 @@ libraryDependencies += "com.itextpdf" % "html2pdf" % "4.0.3"
 // https://mvnrepository.com/artifact/com.github.spullara.mustache.java/compiler
 libraryDependencies += "com.github.spullara.mustache.java" % "compiler" % "0.9.10"
 
+//avro support
+libraryDependencies += "org.apache.avro" % "avro" % "1.11.1"
+
 
 Test / parallelExecution := true
 
@@ -81,16 +84,6 @@ addArtifact(artifact in (Compile, assembly), assembly)
 /***/
 
 
-publishTo := Some(s"GitHub Apache Maven Packages" at s"https://maven.pkg.github.com/${System.getenv("GITHUB_REPOSITORY")}")
-credentials += Credentials(
-  "GitHub Package Registry",
-  "maven.pkg.github.com",
-  System.getenv("GITHUB_REPOSITORY_OWNER"),
-  System.getenv("GITHUB_TOKEN")
-)
-publishMavenStyle := true
-isSnapshot := true
-
 // these are all to allow local publishing to overwrite the version
 publishConfiguration := publishConfiguration.value.withOverwrite(true)
 publishLocalConfiguration := publishLocalConfiguration.value.withOverwrite(true)

diff --git a/data-quality-profiler/publish.sbt b/data-quality-profiler/publish.sbt
@@ -0,0 +1,35 @@
+ThisBuild / organization := "io.github.6point6"
+ThisBuild / organizationName := "6point6"
+ThisBuild / organizationHomepage := Some(url("https://6point6.co.uk/"))
+
+ThisBuild / scmInfo := Some(
+  ScmInfo(
+    url("https://github.com/6point6/data-quality-profiler-and-rules-engine"),
+    "scm:git@github.com:6point6/data-quality-profiler-and-rules-engine.git"
+  )
+)
+ThisBuild / developers := List(
+  Developer(
+    id = "danielsmith-eu",
+    name = "Dr. Daniel Alexander Smith",
+    email = "dan.smith@6point6.co.uk",
+    url = url("http://danielsmith.eu")
+  )
+)
+
+ThisBuild / description := "Data Quality Profiler and Rules Engine."
+ThisBuild / licenses := List(
+  "MIT" -> new URL("https://opensource.org/license/mit/")
+)
+ThisBuild / homepage := Some(url("https://github.com/6point6/data-quality-profiler-and-rules-engine"))
+
+// Remove all additional repository other than Maven Central from POM
+ThisBuild / pomIncludeRepository := { _ => false }
+ThisBuild / publishTo := {
+  // For accounts created after Feb 2021:
+  val nexus = "https://s01.oss.sonatype.org/"
+  //val nexus = "https://oss.sonatype.org/"
+  if (isSnapshot.value) Some("snapshots" at nexus + "content/repositories/snapshots")
+  else Some("releases" at nexus + "service/local/staging/deploy/maven2")
+}
+ThisBuild / publishMavenStyle := true
diff --git a/data-quality-profiler/src/main/scala/uk/gov/ipt/das/dataprofiler/value/RecordValue.scala b/data-quality-profiler/src/main/scala/uk/gov/ipt/das/dataprofiler/value/RecordValue.scala
@@ -86,6 +86,7 @@ object RecordValue {
         case v: java.util.LinkedHashMap[String, Any] => ProfilableRecord.fromLinkedHashMap(linkedHashMap = v)
         case v: java.math.BigDecimal => DoubleValue(v.doubleValue()) // TODO do better, write tests against this
         case v: java.util.ArrayList[_] => ArrayValue(v.asScala.map{ RecordValue.fromAny })
+        case v: org.apache.avro.util.Utf8 => StringValue(v.toString)
         case foo =>
           throw new Exception(s"Unknown primtive type of in RecordValue.fromPrimitive: $foo, class: ${foo.getClass}")
       }

diff --git a/data-quality-profiler/src/test/scala/uk/gov/ipt/das/dataprofiler/value/ValuesTests.scala b/data-quality-profiler/src/test/scala/uk/gov/ipt/das/dataprofiler/value/ValuesTests.scala
@@ -105,6 +105,11 @@ class ValuesTests extends AnyFunSpec {
 
     }
 
+    it("should return a string for a avro record value") {
+      val testValue = RecordValue.fromAny( new org.apache.avro.util.Utf8("foo"))
+      assertResult(testValue.asString)("foo")
+    }
+
   }
 
 }