From 8d60ef4b6cdf178d0ce04419179d63bcfc4090d3 Mon Sep 17 00:00:00 2001 From: David Cavazos Date: Wed, 31 Aug 2022 11:59:09 -0700 Subject: [PATCH] dataflow: replace job name underscores with hyphens (#8303) * dataflow: replace job name underscores with hyphens It looks like Dataflow no longer accepts underscores in the job names. Replacing them with hyphens should work. * fix test checks * improve error reporting * fix test name for exception handling --- dataflow/run_template/main_test.py | 35 ++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/dataflow/run_template/main_test.py b/dataflow/run_template/main_test.py index 0887f1b78179..eb4408b73634 100644 --- a/dataflow/run_template/main_test.py +++ b/dataflow/run_template/main_test.py @@ -66,7 +66,7 @@ def dataflow_job_name(request): # cancel the Dataflow job after running the test # no need to cancel after the empty_args test - it won't create a job and cancellation will throw an error - if label != "test_run_template_empty": + if label != "test-run-template-empty": dataflow_jobs_cancel(job_name) else: print("No active jobs to cancel, cancelling skipped.") @@ -86,12 +86,15 @@ def get_job_id_from_name(job_name): ) response = jobs_request.execute() - # search for the job in the list that has our name (names are unique) - for job in response["jobs"]: - if job["name"] == job_name: - return job["id"] - # if we don't find a job, just return - return + try: + # search for the job in the list that has our name (names are unique) + for job in response["jobs"]: + if job["name"] == job_name: + return job["id"] + # if we don't find a job, just return + return + except Exception as e: + raise ValueError(f"response:\n{response}") from e # We retry the cancel operation a few times until the job is in a state where it can be cancelled @@ -115,7 +118,7 @@ def dataflow_jobs_cancel(job_name): @pytest.mark.parametrize( - "dataflow_job_name", [("test_run_template_empty")], indirect=True + "dataflow_job_name", [("test-run-template-empty")], indirect=True ) def test_run_template_python_empty_args(app, dataflow_job_name): project = PROJECT @@ -125,7 +128,7 @@ def test_run_template_python_empty_args(app, dataflow_job_name): @pytest.mark.parametrize( - "dataflow_job_name", [("test_run_template_python")], indirect=True + "dataflow_job_name", [("test-run-template-python")], indirect=True ) def test_run_template_python(app, dataflow_job_name): project = PROJECT @@ -135,7 +138,7 @@ def test_run_template_python(app, dataflow_job_name): "output": "gs://{}/dataflow/wordcount/outputs".format(BUCKET), } res = main.run(project, dataflow_job_name, template, parameters) - assert "test_run_template_python" in res["job"]["name"] + assert dataflow_job_name in res["job"]["name"] def test_run_template_http_empty_args(app): @@ -145,7 +148,7 @@ def test_run_template_http_empty_args(app): @pytest.mark.parametrize( - "dataflow_job_name", [("test_run_template_url")], indirect=True + "dataflow_job_name", [("test-run-template-url")], indirect=True ) def test_run_template_http_url(app, dataflow_job_name): args = { @@ -158,11 +161,11 @@ def test_run_template_http_url(app, dataflow_job_name): with app.test_request_context("/?" + url_encode(args)): res = main.run_template(flask.request) data = json.loads(res) - assert "test_run_template_url" in data["job"]["name"] + assert dataflow_job_name in data["job"]["name"] @pytest.mark.parametrize( - "dataflow_job_name", [("test_run_template_data")], indirect=True + "dataflow_job_name", [("test-run-template-data")], indirect=True ) def test_run_template_http_data(app, dataflow_job_name): args = { @@ -175,11 +178,11 @@ def test_run_template_http_data(app, dataflow_job_name): with app.test_request_context(data=args): res = main.run_template(flask.request) data = json.loads(res) - assert "test_run_template_data" in data["job"]["name"] + assert dataflow_job_name in data["job"]["name"] @pytest.mark.parametrize( - "dataflow_job_name", [("test_run_template_json")], indirect=True + "dataflow_job_name", [("test-run-template-json")], indirect=True ) def test_run_template_http_json(app, dataflow_job_name): args = { @@ -192,4 +195,4 @@ def test_run_template_http_json(app, dataflow_job_name): with app.test_request_context(json=args): res = main.run_template(flask.request) data = json.loads(res) - assert "test_run_template_json" in data["job"]["name"] + assert dataflow_job_name in data["job"]["name"]