From 1d717a09fd3b173d87fd739c5fcc512bdab3f8fc Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Tue, 29 Mar 2022 17:20:23 +0200 Subject: [PATCH 01/17] # Conflicts: # build.sbt --- build.sbt | 231 ++++++++++-------- .../scala/io/qbeast/core/model/CubeId.scala | 2 +- project/Dependencies.scala | 1 + 3 files changed, 130 insertions(+), 104 deletions(-) diff --git a/build.sbt b/build.sbt index 6824ddbfd..ab28ff2f0 100644 --- a/build.sbt +++ b/build.sbt @@ -2,27 +2,42 @@ import Dependencies._ import xerial.sbt.Sonatype._ lazy val qbeastCore = (project in file("core")) - .settings(name := "qbeast-core", libraryDependencies ++= Seq(apacheCommons % Test)) + .settings( + name := "qbeast-core", + version := "0.1.0", + commonSettings, + releaseSettings, + libraryDependencies ++= Seq(apacheCommons % Test)) + +lazy val qbeastSparkVersion = "0.2.0" + +lazy val qbeastSparkDependencies = + libraryDependencies ++= Seq( + qbeastCoreDep, + deltaCore % Provided, + sparkCore % Provided, + sparkSql % Provided, + hadoopClient % Provided, + sparkFastTests % Test, + amazonAws % Test, + hadoopCommons % Test, + hadoopAws % Test) // Projects lazy val qbeastSpark = (project in file(".")) .enablePlugins(ScalaUnidocPlugin) - .dependsOn(qbeastCore) + .dependsOn(qbeastCore % "test->test;compile->compile;assembly->assembly") .settings( name := "qbeast-spark", - libraryDependencies ++= Seq( - sparkCore % Provided, - sparkSql % Provided, - hadoopClient % Provided, - deltaCore % Provided, - amazonAws % Test, - hadoopCommons % Test, - hadoopAws % Test), + version := qbeastSparkVersion, + commonSettings, + releaseSettings, + qbeastSparkDependencies, + noWarningInConsole, Test / parallelExecution := false, assembly / test := {}, assembly / assemblyOption := (assembly / assemblyOption).value.copy(includeScala = false), publish / skip := true) - .settings(noWarningInConsole) qbeastSpark / Compile / doc / scalacOptions ++= Seq( "-doc-title", @@ -33,54 +48,50 @@ qbeastSpark / Compile / doc / scalacOptions ++= Seq( "Copyright 2022 Qbeast - Docs for version " + qbeast_spark_version + " of qbeast-spark") lazy val qbeastSparkNodep = (project in file("nodep")) - .settings(name := "qbeast-spark-nodep", Compile / packageBin := (qbeastSpark / assembly).value) + .settings( + name := "qbeast-spark-nodep", + commonSettings, + publishGithubSettings, + Compile / packageBin := (qbeastSpark / Compile / packageBin).value) -// As root project has publish / skip := true, we need to create a wrapper project to publish on -// sonatype and Maven Central, which has "qbeast-spark" as name. lazy val qbeastSparkMaven = (project in file("maven")) - .settings(name := "qbeast-spark", Compile / packageBin := (qbeastSpark / assembly).value) - -// Common metadata -val qbeast_spark_version = "0.2.0" -ThisBuild / version := qbeast_spark_version -ThisBuild / organization := "io.qbeast" -ThisBuild / organizationName := "Qbeast Analytics, S.L." -ThisBuild / organizationHomepage := Some(url("https://qbeast.io/")) -ThisBuild / startYear := Some(2021) -ThisBuild / libraryDependencies ++= Seq( - fasterxml % Provided, - sparkFastTests % Test, - scalaTest % Test, - mockito % Test) - -Test / javaOptions += "-Xmx2G" -Test / fork := true - + .settings( + Compile / packageBin := (qbeastSpark / Compile / packageBin).value, + publish / skip := false) + +// COMMON SETTINGS +lazy val commonSettings = Seq( + organization := "io.qbeast", + organizationName := "Qbeast Analytics, S.L.", + organizationHomepage := Some(url("https://qbeast.io/")), + startYear := Some(2021), + libraryDependencies ++= Seq(fasterxml % Provided, scalaTest % Test, mockito % Test), + Test / javaOptions += "-Xmx2G", + Test / fork := true, // Scala compiler settings -ThisBuild / scalaVersion := "2.12.12" -ThisBuild / scalacOptions ++= Seq( - "-encoding", - "UTF-8", - "-target:jvm-1.8", - "-Xfatal-warnings", - "-feature", - "-language:postfixOps", - "-deprecation", - "-unchecked", - "-Xlint") - + scalaVersion := "2.12.12", + scalacOptions ++= Seq( + "-encoding", + "UTF-8", + "-target:jvm-1.8", + "-Xfatal-warnings", + "-feature", + "-language:postfixOps", + "-deprecation", + "-unchecked", + "-Xlint"), // Java compiler settings -ThisBuild / javacOptions ++= Seq( - "-encoding", - "UTF-8", - "-source", - "1.8", - "-target", - "1.8", - "-Werror", - "-g", - "-Xlint", - "-Xdoclint:all/package") + javacOptions ++= Seq( + "-encoding", + "UTF-8", + "-source", + "1.8", + "-target", + "1.8", + "-Werror", + "-g", + "-Xlint", + "-Xdoclint:all/package")) // this setting remove warning when using the sbt console lazy val noWarningInConsole = Seq( @@ -93,14 +104,6 @@ lazy val noWarningInConsole = Seq( // Dependency repositories ThisBuild / resolvers ++= Seq(Resolver.mavenLocal, Resolver.mavenCentral) -// Publication repository (for nightly releases on GitHub) -ThisBuild / publishTo := Some( - "Qbeast Spark" at "https://maven.pkg.github.com/Qbeast-io/qbeast-spark") - -// Repository for maven (Tagged releases on Maven Central using Sonatype) -qbeastSparkMaven / publishTo := sonatypePublishToBundle.value -qbeastSparkMaven / sonatypeCredentialHost := "s01.oss.sonatype.org" - // GitHub Package Registry credentials ThisBuild / credentials += Credentials( "GitHub Package Registry", @@ -113,46 +116,68 @@ ThisBuild / credentials += Credentials( "GHPR_TOKEN", "GHPR_TOKEN required to fetch or publish from/to GitHub Package Registry")) -// Sonatype settings -qbeastSparkMaven / publishMavenStyle := true -qbeastSparkMaven / sonatypeProfileName := "io.qbeast" -qbeastSparkMaven / sonatypeProjectHosting := Some( - GitHubHosting(user = "Qbeast-io", repository = "qbeast-spark", email = "info@qbeast.io")) -qbeastSparkMaven / licenses := Seq( - "APL2" -> url("https://www.apache.org/licenses/LICENSE-2.0.txt")) -qbeastSparkMaven / homepage := Some(url("https://qbeast.io/")) -qbeastSparkMaven / scmInfo := Some( - ScmInfo( - url("https://github.com/Qbeast-io/qbeast-spark"), - "scm:git@github.com:Qbeast-io/qbeast-spark.git")) -qbeastSparkMaven / pomExtra := - - - osopardo1 - Paola Pardo - https://github.com/osopardo1 - - - eavilaes - Eric Avila - https://github.com/eavilaes - - - cugni - Cesare Cugnasco - https://github.com/cugni - - - Jiaweihu08 - Jiawei Hu - https://github.com/Jiaweihu08 - - - alexeiakimov - Alexey Akimov - https://github.com/alexeiakimov - - +lazy val publishGithubSettings = Seq( + // Publication repository (for nightly releases on GitHub) + publishTo := Some("Qbeast Spark" at "https://maven.pkg.github.com/Qbeast-io/qbeast-spark")) + +// RELEASE SETTINGS +lazy val releaseSettings = Seq( + // Repository for maven (Tagged releases on Maven Central using Sonatype) + sonatypeCredentialHost := "s01.oss.sonatype.org", + sonatypeRepository := { + val nexus = "https://s01.oss.sonatype.org/" + if (isSnapshot.value) nexus + "content/repositories/snapshots" + else nexus + "service/local" + }, + publishTo := { + val nexus = "https://s01.oss.sonatype.org/" + if (isSnapshot.value) { + Some("snapshots" at nexus + "content/repositories/snapshots") + } else sonatypePublishToBundle.value + }, + // Sonatype settings + publishMavenStyle := true, + sonatypeProfileName := "io.qbeast", + licenses := Seq("APL2" -> url("https://www.apache.org/licenses/LICENSE-2.0.txt")), + homepage := Some(url("https://qbeast.io/")), + scmInfo := Some( + ScmInfo( + url("https://github.com/Qbeast-io/qbeast-spark"), + "scm:git@github.com:Qbeast-io/qbeast-spark.git")), + pomExtra := + + + Qbeast-io + Qbeast-io + https://github.com/Qbeast-io + info@qbeast.io + + + osopardo1 + Paola Pardo + https://github.com/osopardo1 + + + eavilaes + Eric Avila + https://github.com/eavilaes + + + cugni + Cesare Cugnasco + https://github.com/cugni + + + Jiaweihu08 + Jiawei Hu + https://github.com/Jiaweihu08 + + + alexeiakimov + Alexey Akimov + https://github.com/alexeiakimov + + ) // Scalafmt settings Compile / compile := (Compile / compile).dependsOn(Compile / scalafmtCheck).value diff --git a/core/src/main/scala/io/qbeast/core/model/CubeId.scala b/core/src/main/scala/io/qbeast/core/model/CubeId.scala index a7766ab12..f4ecb9a42 100644 --- a/core/src/main/scala/io/qbeast/core/model/CubeId.scala +++ b/core/src/main/scala/io/qbeast/core/model/CubeId.scala @@ -408,7 +408,7 @@ case class CubeId(dimensionCount: Int, depth: Int, bitMask: Array[Long]) } /** - * Returns the bytes to be stored in a [[org.apache.spark.sql.DataFrame]] + * Returns the bytes to be stored in a org.apache.spark.sql.DataFrame * * @return the bytes */ diff --git a/project/Dependencies.scala b/project/Dependencies.scala index bd04d5880..486b47f75 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -19,4 +19,5 @@ object Dependencies { val hadoopCommons = "org.apache.hadoop" % "hadoop-common" % hadoopVersion val hadoopAws = "org.apache.hadoop" % "hadoop-aws" % hadoopVersion val fasterxml = "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.12.0" + val qbeastCoreDep = "io.qbeast" %% "qbeast-core" % "0.1.0" } From f2bf2f859101e7a304c9b3c87c3fe868c03141c1 Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Tue, 29 Mar 2022 17:25:10 +0200 Subject: [PATCH 02/17] Reorganize build.sbt Separate settings between common, release, and publishGithub Add logic to release SNAPSHOT versions Add qbeast-core:0.1.0 dependency --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index ab28ff2f0..c23a46fd4 100644 --- a/build.sbt +++ b/build.sbt @@ -43,9 +43,9 @@ qbeastSpark / Compile / doc / scalacOptions ++= Seq( "-doc-title", "qbeast-spark", "-doc-version", - qbeast_spark_version, + qbeastSparkVersion, "-doc-footer", - "Copyright 2022 Qbeast - Docs for version " + qbeast_spark_version + " of qbeast-spark") + "Copyright 2022 Qbeast - Docs for version " + qbeastSparkVersion + " of qbeast-spark") lazy val qbeastSparkNodep = (project in file("nodep")) .settings( From 6db7941c6a4357255408214479fc214f35ab1c9a Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Tue, 29 Mar 2022 17:44:58 +0200 Subject: [PATCH 03/17] Leave libraryDependencies inside project module --- build.sbt | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/build.sbt b/build.sbt index c23a46fd4..6de168c5e 100644 --- a/build.sbt +++ b/build.sbt @@ -11,18 +11,6 @@ lazy val qbeastCore = (project in file("core")) lazy val qbeastSparkVersion = "0.2.0" -lazy val qbeastSparkDependencies = - libraryDependencies ++= Seq( - qbeastCoreDep, - deltaCore % Provided, - sparkCore % Provided, - sparkSql % Provided, - hadoopClient % Provided, - sparkFastTests % Test, - amazonAws % Test, - hadoopCommons % Test, - hadoopAws % Test) - // Projects lazy val qbeastSpark = (project in file(".")) .enablePlugins(ScalaUnidocPlugin) @@ -32,7 +20,16 @@ lazy val qbeastSpark = (project in file(".")) version := qbeastSparkVersion, commonSettings, releaseSettings, - qbeastSparkDependencies, + libraryDependencies ++= Seq( + qbeastCoreDep, + deltaCore % Provided, + sparkCore % Provided, + sparkSql % Provided, + hadoopClient % Provided, + sparkFastTests % Test, + amazonAws % Test, + hadoopCommons % Test, + hadoopAws % Test), noWarningInConsole, Test / parallelExecution := false, assembly / test := {}, From be902d49b8873c90b60542fbb5dc7b79c7400996 Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Tue, 29 Mar 2022 17:46:21 +0200 Subject: [PATCH 04/17] Add Compile / doc configuration to module --- build.sbt | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index 6de168c5e..8947a969e 100644 --- a/build.sbt +++ b/build.sbt @@ -31,19 +31,18 @@ lazy val qbeastSpark = (project in file(".")) hadoopCommons % Test, hadoopAws % Test), noWarningInConsole, + Compile / doc / scalacOptions ++= Seq( + "-doc-title", + "qbeast-spark", + "-doc-version", + qbeastSparkVersion, + "-doc-footer", + "Copyright 2022 Qbeast - Docs for version " + qbeastSparkVersion + " of qbeast-spark"), Test / parallelExecution := false, assembly / test := {}, assembly / assemblyOption := (assembly / assemblyOption).value.copy(includeScala = false), publish / skip := true) -qbeastSpark / Compile / doc / scalacOptions ++= Seq( - "-doc-title", - "qbeast-spark", - "-doc-version", - qbeastSparkVersion, - "-doc-footer", - "Copyright 2022 Qbeast - Docs for version " + qbeastSparkVersion + " of qbeast-spark") - lazy val qbeastSparkNodep = (project in file("nodep")) .settings( name := "qbeast-spark-nodep", From 3909f6fb0c08e0d8e23c47cc53842e15ab591541 Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Tue, 29 Mar 2022 18:12:36 +0200 Subject: [PATCH 05/17] Add configuration for qbeastSparkMaven --- build.sbt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.sbt b/build.sbt index 8947a969e..342a84eda 100644 --- a/build.sbt +++ b/build.sbt @@ -46,12 +46,17 @@ lazy val qbeastSpark = (project in file(".")) lazy val qbeastSparkNodep = (project in file("nodep")) .settings( name := "qbeast-spark-nodep", + version := qbeastSparkVersion, commonSettings, publishGithubSettings, Compile / packageBin := (qbeastSpark / Compile / packageBin).value) lazy val qbeastSparkMaven = (project in file("maven")) .settings( + name := "qbeast-spark", + version := qbeastSparkVersion, + commonSettings, + releaseSettings, Compile / packageBin := (qbeastSpark / Compile / packageBin).value, publish / skip := false) From ecc6ecfc4d4b3d5432a87a75099635fa50593035 Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Tue, 29 Mar 2022 18:13:58 +0200 Subject: [PATCH 06/17] Remove qbeastSparkMaven It is not needed since the publish is done manually --- build.sbt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/build.sbt b/build.sbt index 342a84eda..b609d7c51 100644 --- a/build.sbt +++ b/build.sbt @@ -51,15 +51,6 @@ lazy val qbeastSparkNodep = (project in file("nodep")) publishGithubSettings, Compile / packageBin := (qbeastSpark / Compile / packageBin).value) -lazy val qbeastSparkMaven = (project in file("maven")) - .settings( - name := "qbeast-spark", - version := qbeastSparkVersion, - commonSettings, - releaseSettings, - Compile / packageBin := (qbeastSpark / Compile / packageBin).value, - publish / skip := false) - // COMMON SETTINGS lazy val commonSettings = Seq( organization := "io.qbeast", From d3d73ff381b14aece69384873e0a8833e0f6e4df Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Wed, 30 Mar 2022 10:05:32 +0200 Subject: [PATCH 07/17] Fix sbt clean and rollback to assembly in nodep packaging --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index b609d7c51..9031541ec 100644 --- a/build.sbt +++ b/build.sbt @@ -14,7 +14,7 @@ lazy val qbeastSparkVersion = "0.2.0" // Projects lazy val qbeastSpark = (project in file(".")) .enablePlugins(ScalaUnidocPlugin) - .dependsOn(qbeastCore % "test->test;compile->compile;assembly->assembly") + .dependsOn(qbeastCore) .settings( name := "qbeast-spark", version := qbeastSparkVersion, @@ -49,7 +49,7 @@ lazy val qbeastSparkNodep = (project in file("nodep")) version := qbeastSparkVersion, commonSettings, publishGithubSettings, - Compile / packageBin := (qbeastSpark / Compile / packageBin).value) + Compile / packageBin := (qbeastSpark / assembly).value) // COMMON SETTINGS lazy val commonSettings = Seq( From ea211fe3e06984d5238dabb187af8522d3e65b76 Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Wed, 30 Mar 2022 10:21:30 +0200 Subject: [PATCH 08/17] Revert to qbeastSparkMaven --- build.sbt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 9031541ec..cf1740728 100644 --- a/build.sbt +++ b/build.sbt @@ -19,7 +19,6 @@ lazy val qbeastSpark = (project in file(".")) name := "qbeast-spark", version := qbeastSparkVersion, commonSettings, - releaseSettings, libraryDependencies ++= Seq( qbeastCoreDep, deltaCore % Provided, @@ -51,6 +50,14 @@ lazy val qbeastSparkNodep = (project in file("nodep")) publishGithubSettings, Compile / packageBin := (qbeastSpark / assembly).value) +lazy val qbeastSparkMaven = (project in file("maven")) + .settings( + name := "qbeast-spark", + version := qbeastSparkVersion, + commonSettings, + releaseSettings, + Compile / packageBin := (qbeastSpark / Compile / packageBin).value) + // COMMON SETTINGS lazy val commonSettings = Seq( organization := "io.qbeast", From 2941125ab586bee6cba6716aa42b548eae46d05d Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Wed, 30 Mar 2022 11:48:17 +0200 Subject: [PATCH 09/17] Add dependencies to qbeastSparkMaven --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index cf1740728..5a4f62023 100644 --- a/build.sbt +++ b/build.sbt @@ -56,6 +56,7 @@ lazy val qbeastSparkMaven = (project in file("maven")) version := qbeastSparkVersion, commonSettings, releaseSettings, + libraryDependencies := (qbeastSpark / libraryDependencies).value, Compile / packageBin := (qbeastSpark / Compile / packageBin).value) // COMMON SETTINGS From 7145b0856e1911561cfc72beac846d624bbad94b Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Thu, 31 Mar 2022 14:10:00 +0200 Subject: [PATCH 10/17] Remove dependency, revert to ThisBuild --- build.sbt | 225 +++++++++++++++++-------------------- project/Dependencies.scala | 1 - 2 files changed, 103 insertions(+), 123 deletions(-) diff --git a/build.sbt b/build.sbt index 5a4f62023..cee602cba 100644 --- a/build.sbt +++ b/build.sbt @@ -5,93 +5,83 @@ lazy val qbeastCore = (project in file("core")) .settings( name := "qbeast-core", version := "0.1.0", - commonSettings, - releaseSettings, libraryDependencies ++= Seq(apacheCommons % Test)) -lazy val qbeastSparkVersion = "0.2.0" - // Projects lazy val qbeastSpark = (project in file(".")) .enablePlugins(ScalaUnidocPlugin) .dependsOn(qbeastCore) .settings( name := "qbeast-spark", - version := qbeastSparkVersion, - commonSettings, libraryDependencies ++= Seq( - qbeastCoreDep, - deltaCore % Provided, sparkCore % Provided, sparkSql % Provided, hadoopClient % Provided, - sparkFastTests % Test, + deltaCore % Provided, amazonAws % Test, hadoopCommons % Test, hadoopAws % Test), - noWarningInConsole, - Compile / doc / scalacOptions ++= Seq( - "-doc-title", - "qbeast-spark", - "-doc-version", - qbeastSparkVersion, - "-doc-footer", - "Copyright 2022 Qbeast - Docs for version " + qbeastSparkVersion + " of qbeast-spark"), Test / parallelExecution := false, assembly / test := {}, assembly / assemblyOption := (assembly / assemblyOption).value.copy(includeScala = false), publish / skip := true) + .settings(noWarningInConsole) + +qbeastSpark / Compile / doc / scalacOptions ++= Seq( + "-doc-title", + "qbeast-spark", + "-doc-version", + qbeast_spark_version, + "-doc-footer", + "Copyright 2022 Qbeast - Docs for version " + qbeast_spark_version + " of qbeast-spark") lazy val qbeastSparkNodep = (project in file("nodep")) .settings( name := "qbeast-spark-nodep", - version := qbeastSparkVersion, - commonSettings, - publishGithubSettings, + publishTo := Some("Qbeast Spark" at "https://maven.pkg.github.com/Qbeast-io/qbeast-spark"), Compile / packageBin := (qbeastSpark / assembly).value) -lazy val qbeastSparkMaven = (project in file("maven")) - .settings( - name := "qbeast-spark", - version := qbeastSparkVersion, - commonSettings, - releaseSettings, - libraryDependencies := (qbeastSpark / libraryDependencies).value, - Compile / packageBin := (qbeastSpark / Compile / packageBin).value) - -// COMMON SETTINGS -lazy val commonSettings = Seq( - organization := "io.qbeast", - organizationName := "Qbeast Analytics, S.L.", - organizationHomepage := Some(url("https://qbeast.io/")), - startYear := Some(2021), - libraryDependencies ++= Seq(fasterxml % Provided, scalaTest % Test, mockito % Test), - Test / javaOptions += "-Xmx2G", - Test / fork := true, +// Common metadata +val qbeast_spark_version = "0.2.0" +ThisBuild / version := qbeast_spark_version +ThisBuild / organization := "io.qbeast" +ThisBuild / organizationName := "Qbeast Analytics, S.L." +ThisBuild / organizationHomepage := Some(url("https://qbeast.io/")) +ThisBuild / startYear := Some(2021) +ThisBuild / libraryDependencies ++= Seq( + fasterxml % Provided, + sparkFastTests % Test, + scalaTest % Test, + mockito % Test) + +Test / javaOptions += "-Xmx2G" +Test / fork := true + // Scala compiler settings - scalaVersion := "2.12.12", - scalacOptions ++= Seq( - "-encoding", - "UTF-8", - "-target:jvm-1.8", - "-Xfatal-warnings", - "-feature", - "-language:postfixOps", - "-deprecation", - "-unchecked", - "-Xlint"), +ThisBuild / scalaVersion := "2.12.12" +ThisBuild / scalacOptions ++= Seq( + "-encoding", + "UTF-8", + "-target:jvm-1.8", + "-Xfatal-warnings", + "-feature", + "-language:postfixOps", + "-deprecation", + "-unchecked", + "-Xlint") + // Java compiler settings - javacOptions ++= Seq( - "-encoding", - "UTF-8", - "-source", - "1.8", - "-target", - "1.8", - "-Werror", - "-g", - "-Xlint", - "-Xdoclint:all/package")) +ThisBuild / javacOptions ++= Seq( + "-encoding", + "UTF-8", + "-source", + "1.8", + "-target", + "1.8", + "-Werror", + "-g", + "-Xlint", + "-Xdoclint:all/package") // this setting remove warning when using the sbt console lazy val noWarningInConsole = Seq( @@ -104,6 +94,20 @@ lazy val noWarningInConsole = Seq( // Dependency repositories ThisBuild / resolvers ++= Seq(Resolver.mavenLocal, Resolver.mavenCentral) +// Repository for maven (Tagged releases on Maven Central using Sonatype) +ThisBuild / sonatypeCredentialHost := "s01.oss.sonatype.org" +ThisBuild / sonatypeRepository := { + val nexus = "https://s01.oss.sonatype.org/" + if (isSnapshot.value) nexus + "content/repositories/snapshots" + else nexus + "service/local" +} +ThisBuild / publishTo := { + val nexus = "https://s01.oss.sonatype.org/" + if (isSnapshot.value) { + Some("snapshots" at nexus + "content/repositories/snapshots") + } else sonatypePublishToBundle.value +} + // GitHub Package Registry credentials ThisBuild / credentials += Credentials( "GitHub Package Registry", @@ -116,68 +120,45 @@ ThisBuild / credentials += Credentials( "GHPR_TOKEN", "GHPR_TOKEN required to fetch or publish from/to GitHub Package Registry")) -lazy val publishGithubSettings = Seq( - // Publication repository (for nightly releases on GitHub) - publishTo := Some("Qbeast Spark" at "https://maven.pkg.github.com/Qbeast-io/qbeast-spark")) - -// RELEASE SETTINGS -lazy val releaseSettings = Seq( - // Repository for maven (Tagged releases on Maven Central using Sonatype) - sonatypeCredentialHost := "s01.oss.sonatype.org", - sonatypeRepository := { - val nexus = "https://s01.oss.sonatype.org/" - if (isSnapshot.value) nexus + "content/repositories/snapshots" - else nexus + "service/local" - }, - publishTo := { - val nexus = "https://s01.oss.sonatype.org/" - if (isSnapshot.value) { - Some("snapshots" at nexus + "content/repositories/snapshots") - } else sonatypePublishToBundle.value - }, - // Sonatype settings - publishMavenStyle := true, - sonatypeProfileName := "io.qbeast", - licenses := Seq("APL2" -> url("https://www.apache.org/licenses/LICENSE-2.0.txt")), - homepage := Some(url("https://qbeast.io/")), - scmInfo := Some( - ScmInfo( - url("https://github.com/Qbeast-io/qbeast-spark"), - "scm:git@github.com:Qbeast-io/qbeast-spark.git")), - pomExtra := - - - Qbeast-io - Qbeast-io - https://github.com/Qbeast-io - info@qbeast.io - - - osopardo1 - Paola Pardo - https://github.com/osopardo1 - - - eavilaes - Eric Avila - https://github.com/eavilaes - - - cugni - Cesare Cugnasco - https://github.com/cugni - - - Jiaweihu08 - Jiawei Hu - https://github.com/Jiaweihu08 - - - alexeiakimov - Alexey Akimov - https://github.com/alexeiakimov - - ) +// Sonatype settings +ThisBuild / publishMavenStyle := true +ThisBuild / sonatypeProfileName := "io.qbeast" +ThisBuild / sonatypeProjectHosting := Some( + GitHubHosting(user = "Qbeast-io", repository = "qbeast-spark", email = "info@qbeast.io")) +ThisBuild / licenses := Seq("APL2" -> url("https://www.apache.org/licenses/LICENSE-2.0.txt")) +ThisBuild / homepage := Some(url("https://qbeast.io/")) +ThisBuild / scmInfo := Some( + ScmInfo( + url("https://github.com/Qbeast-io/qbeast-spark"), + "scm:git@github.com:Qbeast-io/qbeast-spark.git")) +ThisBuild / pomExtra := + + + osopardo1 + Paola Pardo + https://github.com/osopardo1 + + + eavilaes + Eric Avila + https://github.com/eavilaes + + + cugni + Cesare Cugnasco + https://github.com/cugni + + + Jiaweihu08 + Jiawei Hu + https://github.com/Jiaweihu08 + + + alexeiakimov + Alexey Akimov + https://github.com/alexeiakimov + + // Scalafmt settings Compile / compile := (Compile / compile).dependsOn(Compile / scalafmtCheck).value diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 486b47f75..bd04d5880 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -19,5 +19,4 @@ object Dependencies { val hadoopCommons = "org.apache.hadoop" % "hadoop-common" % hadoopVersion val hadoopAws = "org.apache.hadoop" % "hadoop-aws" % hadoopVersion val fasterxml = "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.12.0" - val qbeastCoreDep = "io.qbeast" %% "qbeast-core" % "0.1.0" } From 04428ff8f035d6f9c365243ea6f96fe7feff1b22 Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Thu, 31 Mar 2022 14:11:32 +0200 Subject: [PATCH 11/17] Remove spark reference in CubeId --- core/src/main/scala/io/qbeast/core/model/CubeId.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/io/qbeast/core/model/CubeId.scala b/core/src/main/scala/io/qbeast/core/model/CubeId.scala index f4ecb9a42..a75d5f9d5 100644 --- a/core/src/main/scala/io/qbeast/core/model/CubeId.scala +++ b/core/src/main/scala/io/qbeast/core/model/CubeId.scala @@ -408,7 +408,7 @@ case class CubeId(dimensionCount: Int, depth: Int, bitMask: Array[Long]) } /** - * Returns the bytes to be stored in a org.apache.spark.sql.DataFrame + * Returns the bytes to be stored in a byte array. * * @return the bytes */ From d291fcb0e61712f1432d9d7291c4cd874fcff840 Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Fri, 1 Apr 2022 11:22:27 +0200 Subject: [PATCH 12/17] Remove publish of fat jar in Github registry --- ...publish-artifact.yml => test-artifact.yml} | 11 ++-------- build.sbt | 21 ++----------------- 2 files changed, 4 insertions(+), 28 deletions(-) rename .github/workflows/{test-and-publish-artifact.yml => test-artifact.yml} (70%) diff --git a/.github/workflows/test-and-publish-artifact.yml b/.github/workflows/test-artifact.yml similarity index 70% rename from .github/workflows/test-and-publish-artifact.yml rename to .github/workflows/test-artifact.yml index 405a60f58..662ca5392 100644 --- a/.github/workflows/test-and-publish-artifact.yml +++ b/.github/workflows/test-artifact.yml @@ -1,11 +1,11 @@ -name: Test and publish artifact +name: Test artifact on: push: branches: [ main ] pull_request: jobs: - test-and-publish-artifact: + test-artifact: runs-on: ubuntu-latest env: GHPR_TOKEN: ${{ secrets.GHPR_TOKEN }} @@ -33,10 +33,3 @@ jobs: ./codecov -t ${{ secrets.CODECOV_TOKEN }} - name: Formatting of code and Scaladocs run: sbt scalafmtSbtCheck doc - - # Publish only if it is on the main branch - - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'Qbeast-io/qbeast-spark' }} - name: Publish artifact - run: | - QBEAST_SPARK_VERSION="nightly-$(git rev-parse --short ${GITHUB_SHA})" - sbt "set qbeastSparkNodep/version := \"${QBEAST_SPARK_VERSION}\"" qbeastSparkNodep/publish diff --git a/build.sbt b/build.sbt index cee602cba..a8eabe7b4 100644 --- a/build.sbt +++ b/build.sbt @@ -7,6 +7,8 @@ lazy val qbeastCore = (project in file("core")) version := "0.1.0", libraryDependencies ++= Seq(apacheCommons % Test)) +val qbeast_spark_version = "0.2.0" + // Projects lazy val qbeastSpark = (project in file(".")) .enablePlugins(ScalaUnidocPlugin) @@ -35,14 +37,7 @@ qbeastSpark / Compile / doc / scalacOptions ++= Seq( "-doc-footer", "Copyright 2022 Qbeast - Docs for version " + qbeast_spark_version + " of qbeast-spark") -lazy val qbeastSparkNodep = (project in file("nodep")) - .settings( - name := "qbeast-spark-nodep", - publishTo := Some("Qbeast Spark" at "https://maven.pkg.github.com/Qbeast-io/qbeast-spark"), - Compile / packageBin := (qbeastSpark / assembly).value) - // Common metadata -val qbeast_spark_version = "0.2.0" ThisBuild / version := qbeast_spark_version ThisBuild / organization := "io.qbeast" ThisBuild / organizationName := "Qbeast Analytics, S.L." @@ -108,18 +103,6 @@ ThisBuild / publishTo := { } else sonatypePublishToBundle.value } -// GitHub Package Registry credentials -ThisBuild / credentials += Credentials( - "GitHub Package Registry", - "maven.pkg.github.com", - sys.env.getOrElse( - // Any user will work if you're using a TOKEN as password - "GHPR_USERNAME", - "GHPR_USERNAME required to fetch or publish from/to GH Package Registry"), - sys.env.getOrElse( - "GHPR_TOKEN", - "GHPR_TOKEN required to fetch or publish from/to GitHub Package Registry")) - // Sonatype settings ThisBuild / publishMavenStyle := true ThisBuild / sonatypeProfileName := "io.qbeast" From 43cfe1e1df98ffe76ea625407923556f9cb7ce52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20=C3=81vila?= Date: Fri, 1 Apr 2022 11:37:22 +0200 Subject: [PATCH 13/17] Remove Dockerfile --- .github/workflows/publish-container.yml | 33 ------------------------- Dockerfile | 13 ---------- 2 files changed, 46 deletions(-) delete mode 100644 .github/workflows/publish-container.yml delete mode 100644 Dockerfile diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml deleted file mode 100644 index 6f96ed55a..000000000 --- a/.github/workflows/publish-container.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Publish Container image -on: - push: - branches: [ main ] - paths: - - 'Dockerfile' - pull_request: - paths: - - 'Dockerfile' - workflow_dispatch: -jobs: - publish-container: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build docker image - run: | - CONTAINER_VERSION="nightly-$(git rev-parse --short ${GITHUB_SHA})" - docker build . --rm --force-rm -t ghcr.io/qbeast-io/qbeast-spark/qbeast-spark:${CONTAINER_VERSION} - - # Publish only if it is on the main branch - - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Log into registry - run: docker login ghcr.io -u "Any User" -p ${{ secrets.GITHUB_TOKEN }} - - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Publish docker image - run: | - QBEAST_SPARK_VERSION="nightly-$(git rev-parse --short ${GITHUB_SHA})" - docker push ghcr.io/qbeast-io/qbeast-spark/qbeast-spark:${QBEAST_SPARK_VERSION} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f045bd54f..000000000 --- a/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM gcr.io/spark-operator/spark:v3.1.1 -USER root - -ADD https://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/2.0.0/azure-storage-2.0.0.jar \ - https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.2.0/hadoop-azure-3.2.0.jar \ - https://repo1.maven.org/maven2/com/azure/azure-storage-blob/12.8.0/azure-storage-blob-12.8.0.jar \ - https://repo1.maven.org/maven2/com/azure/azure-storage-common/12.8.0/azure-storage-common-12.8.0.jar \ - https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.0/hadoop-aws-3.2.0.jar \ - https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar \ - https://repo1.maven.org/maven2/io/delta/delta-core_2.12/0.8.0/delta-core_2.12-0.8.0.jar \ - https://repo1.maven.org/maven2/com/google/guava/guava/18.0/guava-18.0.jar \ - /opt/spark/jars/ - From 7affce456d289862d05da8f6240ec104cec60ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20=C3=81vila?= Date: Fri, 1 Apr 2022 11:38:04 +0200 Subject: [PATCH 14/17] Split qbeastCore and qbeastSpark projects' tests --- .github/workflows/test-qbeastCore.yml | 32 +++++++++++++++++++ ...test-artifact.yml => test-qbeastSpark.yml} | 13 ++++---- 2 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/test-qbeastCore.yml rename .github/workflows/{test-artifact.yml => test-qbeastSpark.yml} (81%) diff --git a/.github/workflows/test-qbeastCore.yml b/.github/workflows/test-qbeastCore.yml new file mode 100644 index 000000000..b41c66792 --- /dev/null +++ b/.github/workflows/test-qbeastCore.yml @@ -0,0 +1,32 @@ +name: Test qbeastCore project +on: + push: + branches: [ main ] + pull_request: + +jobs: + test-artifact: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Package qbeast-core + run: sbt "project qbeastCore" clean package + - name: Test + run: | + sbt "project qbeastCore" coverage 'test' coverageReport + - name: Upload qbeastCore coverage to Codecov + run: | + curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import # One-time step + curl -Os https://uploader.codecov.io/latest/linux/codecov + curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM + curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig + gpgv codecov.SHA256SUM.sig codecov.SHA256SUM + shasum -a 256 -c codecov.SHA256SUM + chmod +x codecov + ./codecov -t ${{ secrets.CODECOV_TOKEN }} + - name: Formatting of code and Scaladocs + run: sbt "project qbeastCore" scalafmtSbtCheck doc diff --git a/.github/workflows/test-artifact.yml b/.github/workflows/test-qbeastSpark.yml similarity index 81% rename from .github/workflows/test-artifact.yml rename to .github/workflows/test-qbeastSpark.yml index 662ca5392..c5f220dd5 100644 --- a/.github/workflows/test-artifact.yml +++ b/.github/workflows/test-qbeastSpark.yml @@ -1,4 +1,4 @@ -name: Test artifact +name: Test qbeastSpark project on: push: branches: [ main ] @@ -7,21 +7,18 @@ on: jobs: test-artifact: runs-on: ubuntu-latest - env: - GHPR_TOKEN: ${{ secrets.GHPR_TOKEN }} steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Assembly - run: sbt assembly + - name: Package qbeast-spark + run: sbt clean package - name: Test run: | sbt coverage 'test' coverageReport - sbt "project qbeastCore" coverage 'test' coverageReport - - name: Upload to Codecov + - name: Upload qbeastSpark coverage to Codecov run: | curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import # One-time step curl -Os https://uploader.codecov.io/latest/linux/codecov @@ -33,3 +30,5 @@ jobs: ./codecov -t ${{ secrets.CODECOV_TOKEN }} - name: Formatting of code and Scaladocs run: sbt scalafmtSbtCheck doc + - name: Run unidoc for generating docs + run: sbt unidoc From 4b9b948428d1e3d3d5dde8f4c6453096410a9b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20=C3=81vila?= Date: Fri, 1 Apr 2022 12:05:31 +0200 Subject: [PATCH 15/17] Update deprecated syntax --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index a8eabe7b4..e60891ecf 100644 --- a/build.sbt +++ b/build.sbt @@ -80,11 +80,11 @@ ThisBuild / javacOptions ++= Seq( // this setting remove warning when using the sbt console lazy val noWarningInConsole = Seq( - scalacOptions in (Compile, console) ~= { + Compile / console / scalacOptions ~= { _.filterNot( Set("-Ywarn-unused-import", "-Ywarn-unused:imports", "-Xlint", "-Xfatal-warnings")) }, - scalacOptions in (Test, console) := (scalacOptions in (Compile, console)).value) + Test / console / scalacOptions := (Compile / console / scalacOptions).value) // Dependency repositories ThisBuild / resolvers ++= Seq(Resolver.mavenLocal, Resolver.mavenCentral) From 3851146142ed32ce119c97d4809e8e01b51c61c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20=C3=81vila?= Date: Fri, 1 Apr 2022 11:38:04 +0200 Subject: [PATCH 16/17] Revert "Split qbeastCore and qbeastSpark projects' tests" This reverts commit 7affce456d289862d05da8f6240ec104cec60ee0. We found a problem with Codecov coverage, so we are reverting this commit until we find a solution. --- ...test-qbeastSpark.yml => test-artifact.yml} | 13 ++++---- .github/workflows/test-qbeastCore.yml | 32 ------------------- 2 files changed, 7 insertions(+), 38 deletions(-) rename .github/workflows/{test-qbeastSpark.yml => test-artifact.yml} (81%) delete mode 100644 .github/workflows/test-qbeastCore.yml diff --git a/.github/workflows/test-qbeastSpark.yml b/.github/workflows/test-artifact.yml similarity index 81% rename from .github/workflows/test-qbeastSpark.yml rename to .github/workflows/test-artifact.yml index c5f220dd5..662ca5392 100644 --- a/.github/workflows/test-qbeastSpark.yml +++ b/.github/workflows/test-artifact.yml @@ -1,4 +1,4 @@ -name: Test qbeastSpark project +name: Test artifact on: push: branches: [ main ] @@ -7,18 +7,21 @@ on: jobs: test-artifact: runs-on: ubuntu-latest + env: + GHPR_TOKEN: ${{ secrets.GHPR_TOKEN }} steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Package qbeast-spark - run: sbt clean package + - name: Assembly + run: sbt assembly - name: Test run: | sbt coverage 'test' coverageReport - - name: Upload qbeastSpark coverage to Codecov + sbt "project qbeastCore" coverage 'test' coverageReport + - name: Upload to Codecov run: | curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import # One-time step curl -Os https://uploader.codecov.io/latest/linux/codecov @@ -30,5 +33,3 @@ jobs: ./codecov -t ${{ secrets.CODECOV_TOKEN }} - name: Formatting of code and Scaladocs run: sbt scalafmtSbtCheck doc - - name: Run unidoc for generating docs - run: sbt unidoc diff --git a/.github/workflows/test-qbeastCore.yml b/.github/workflows/test-qbeastCore.yml deleted file mode 100644 index b41c66792..000000000 --- a/.github/workflows/test-qbeastCore.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Test qbeastCore project -on: - push: - branches: [ main ] - pull_request: - -jobs: - test-artifact: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Package qbeast-core - run: sbt "project qbeastCore" clean package - - name: Test - run: | - sbt "project qbeastCore" coverage 'test' coverageReport - - name: Upload qbeastCore coverage to Codecov - run: | - curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import # One-time step - curl -Os https://uploader.codecov.io/latest/linux/codecov - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig - gpgv codecov.SHA256SUM.sig codecov.SHA256SUM - shasum -a 256 -c codecov.SHA256SUM - chmod +x codecov - ./codecov -t ${{ secrets.CODECOV_TOKEN }} - - name: Formatting of code and Scaladocs - run: sbt "project qbeastCore" scalafmtSbtCheck doc From 5fdbf07efa7d6a56e16296a5c1423338f57bc066 Mon Sep 17 00:00:00 2001 From: osopardo1 Date: Tue, 5 Apr 2022 12:54:48 +0200 Subject: [PATCH 17/17] Change order of contributors --- build.sbt | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/build.sbt b/build.sbt index e60891ecf..cfb53b36a 100644 --- a/build.sbt +++ b/build.sbt @@ -117,29 +117,34 @@ ThisBuild / scmInfo := Some( ThisBuild / pomExtra := - osopardo1 - Paola Pardo - https://github.com/osopardo1 - - - eavilaes - Eric Avila - https://github.com/eavilaes + alexeiakimov + Alexey Akimov + https://github.com/alexeiakimov cugni Cesare Cugnasco https://github.com/cugni + + eavilaes + Eric Avila + https://github.com/eavilaes + Jiaweihu08 Jiawei Hu https://github.com/Jiaweihu08 - alexeiakimov - Alexey Akimov - https://github.com/alexeiakimov + osopardo1 + Paola Pardo + https://github.com/osopardo1 + + + polsm91 + Pol Santamaria + https://github.com/polsm91