-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathprocess.py
33 lines (23 loc) · 1 KB
/
process.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions, StandardOptions
import os
from apache_beam import window
# Replace 'my-service-account-path' with your service account path
service_account_path = 'my-service-account-path'
print("Service account file : ", service_account_path)
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_path
# Replace 'my-input-subscription' with your input subscription id
input_subscription = 'my-input-subscription'
# Replace 'my-output-subscription' with your output subscription id
output_topic = 'my-output-subscription'
options = PipelineOptions()
options.view_as(StandardOptions).streaming = True
p = beam.Pipeline(options=options)
output_file = 'outputs/part'
pubsub_data = (
p
| 'Read from pub sub' >> beam.io.ReadFromPubSub(subscription= input_subscription)
| 'Write to pus sub' >> beam.io.WriteToPubSub(output_topic)
)
result = p.run()
result.wait_until_finish()