From 662e1a60ac82a1c6a811cbb257661b66169d6cd5 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 20 Oct 2021 18:41:40 -0400 Subject: [PATCH 1/4] chore(deps): update Spark version+nits --- java/pom.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index dde84d9..05f25c3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -81,12 +81,13 @@ 3.2.0 3.5.1 2.4 + 4.4.2 2.2 0.12 - 2.3.2 + 2.4.7 2.11 - v1-rev403-1.25.0 + v1-rev452-1.25.0 2.9.7 3.8.1 4.2 @@ -355,7 +356,7 @@ org.apache.rat apache-rat-plugin - ${apapache.maven.rat.version} + ${apache.maven.rat.version} en @@ -453,4 +454,4 @@ s3://maven.signifai.io/release - \ No newline at end of file + From 0a468b2893d907ae5792add35efe7bd4d49c95eb Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 20 Oct 2021 18:42:38 -0400 Subject: [PATCH 2/4] docs: set decodeData to true in message receiver --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5bbd738..bacc982 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ import org.apache.spark.rdd._ val SUBSCRIPTION = "" var ssc = new StreamingContext(sc,Seconds(5)) -var pubsubReceiver = new PubsubReceiver(SUBSCRIPTION, 10) +var pubsubReceiver = new PubsubReceiver(SUBSCRIPTION, 10, true) val customReceiverStream = ssc.receiverStream(pubsubReceiver) customReceiverStream.map(x => x).foreachRDD((x: RDD[String]) => println(x.count)) customReceiverStream.map(x => x).foreachRDD((x: RDD[String]) => println(x)) @@ -85,7 +85,7 @@ from signifai.pubsub import PubsubUtils SUBSCRIPTION = "" ssc = StreamingContext(sc, 1) -pubsubStream = PubsubUtils.createStream(ssc, SUBSCRIPTION, 5) +pubsubStream = PubsubUtils.createStream(ssc, SUBSCRIPTION, 5, True) pubsubStream.flatMap(lambda x: x).pprint() ssc.start() ``` From 398be83a99e4ad5110ecd840bc7822fb1387d192 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 21 Oct 2021 11:17:42 -0700 Subject: [PATCH 3/4] update Spark to 2.4.8 --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 05f25c3..36966af 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -85,7 +85,7 @@ 2.2 0.12 - 2.4.7 + 2.4.8 2.11 v1-rev452-1.25.0 2.9.7 From ee7284d15fbb665c7902f36f5fb5e183b6db10b6 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 21 Oct 2021 16:38:01 -0700 Subject: [PATCH 4/4] chore(deps): update scala to 2.12 --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 36966af..a62e255 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -86,7 +86,7 @@ 0.12 2.4.8 - 2.11 + 2.12 v1-rev452-1.25.0 2.9.7 3.8.1