adding spark job example

Dhanush123 · Mar 30, 2020 · a49dd0c · a49dd0c
1 parent beec15e
commit a49dd0c
Show file tree

Hide file tree

Showing 6 changed files with 8 additions and 35 deletions.
diff --git a/.dockerignore b/.dockerignore
diff --git a/Dockerfile b/Dockerfile
diff --git a/Pipfile b/Pipfile
@@ -8,4 +8,4 @@ verify_ssl = true
 [packages]
 
 [requires]
-python_version = "3.8"
+python_version = "3.7"
diff --git a/mnist_model.joblib b/mnist_model.joblib
diff --git a/mnist_model.pkl b/mnist_model.pkl
diff --git a/testspark.py b/testspark.py
@@ -0,0 +1,7 @@
+import pyspark
+
+sc = pyspark.SparkContext()
+rdd = sc.parallelize(['Hello,', 'world!', 'dog', 'elephant', 'panther'])
+res = rdd.collect()
+print("res", res)
+rdd.coalesce(1).saveAsTextFile(SPARK_BUCKET_OUTPUT_PATH)