Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dataflow: replace job name underscores with hyphens #8303

Merged
merged 5 commits into from
Aug 31, 2022
Merged

dataflow: replace job name underscores with hyphens #8303

Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
15496f4
dataflow: replace job name underscores with hyphens
davidcavazos Aug 31, 2022
c0f4313
fix test checks
davidcavazos Aug 31, 2022
f7845f5
improve error reporting
davidcavazos Aug 31, 2022
10bf31b
fix test name for exception handling
davidcavazos Aug 31, 2022
0a86658
Merge branch 'main' into dataflow-name-hyphens
davidcavazos Aug 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 19 additions & 16 deletions dataflow/run_template/main_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def dataflow_job_name(request):

# cancel the Dataflow job after running the test
# no need to cancel after the empty_args test - it won't create a job and cancellation will throw an error
if label != "test_run_template_empty":
if label != "test-run-template-empty":
dataflow_jobs_cancel(job_name)
else:
print("No active jobs to cancel, cancelling skipped.")
Expand All @@ -86,12 +86,15 @@ def get_job_id_from_name(job_name):
)
response = jobs_request.execute()

# search for the job in the list that has our name (names are unique)
for job in response["jobs"]:
if job["name"] == job_name:
return job["id"]
# if we don't find a job, just return
return
try:
# search for the job in the list that has our name (names are unique)
for job in response["jobs"]:
if job["name"] == job_name:
return job["id"]
# if we don't find a job, just return
return
except Exception as e:
raise ValueError(f"response:\n{response}") from e


# We retry the cancel operation a few times until the job is in a state where it can be cancelled
Expand All @@ -115,7 +118,7 @@ def dataflow_jobs_cancel(job_name):


@pytest.mark.parametrize(
"dataflow_job_name", [("test_run_template_empty")], indirect=True
"dataflow_job_name", [("test-run-template-empty")], indirect=True
)
def test_run_template_python_empty_args(app, dataflow_job_name):
project = PROJECT
Expand All @@ -125,7 +128,7 @@ def test_run_template_python_empty_args(app, dataflow_job_name):


@pytest.mark.parametrize(
"dataflow_job_name", [("test_run_template_python")], indirect=True
"dataflow_job_name", [("test-run-template-python")], indirect=True
)
def test_run_template_python(app, dataflow_job_name):
project = PROJECT
Expand All @@ -135,7 +138,7 @@ def test_run_template_python(app, dataflow_job_name):
"output": "gs://{}/dataflow/wordcount/outputs".format(BUCKET),
}
res = main.run(project, dataflow_job_name, template, parameters)
assert "test_run_template_python" in res["job"]["name"]
assert dataflow_job_name in res["job"]["name"]


def test_run_template_http_empty_args(app):
Expand All @@ -145,7 +148,7 @@ def test_run_template_http_empty_args(app):


@pytest.mark.parametrize(
"dataflow_job_name", [("test_run_template_url")], indirect=True
"dataflow_job_name", [("test-run-template-url")], indirect=True
)
def test_run_template_http_url(app, dataflow_job_name):
args = {
Expand All @@ -158,11 +161,11 @@ def test_run_template_http_url(app, dataflow_job_name):
with app.test_request_context("/?" + url_encode(args)):
res = main.run_template(flask.request)
data = json.loads(res)
assert "test_run_template_url" in data["job"]["name"]
assert dataflow_job_name in data["job"]["name"]


@pytest.mark.parametrize(
"dataflow_job_name", [("test_run_template_data")], indirect=True
"dataflow_job_name", [("test-run-template-data")], indirect=True
)
def test_run_template_http_data(app, dataflow_job_name):
args = {
Expand All @@ -175,11 +178,11 @@ def test_run_template_http_data(app, dataflow_job_name):
with app.test_request_context(data=args):
res = main.run_template(flask.request)
data = json.loads(res)
assert "test_run_template_data" in data["job"]["name"]
assert dataflow_job_name in data["job"]["name"]


@pytest.mark.parametrize(
"dataflow_job_name", [("test_run_template_json")], indirect=True
"dataflow_job_name", [("test-run-template-json")], indirect=True
)
def test_run_template_http_json(app, dataflow_job_name):
args = {
Expand All @@ -192,4 +195,4 @@ def test_run_template_http_json(app, dataflow_job_name):
with app.test_request_context(json=args):
res = main.run_template(flask.request)
data = json.loads(res)
assert "test_run_template_json" in data["job"]["name"]
assert dataflow_job_name in data["job"]["name"]