From 2cecff8814c5f9279aa24a77c107fc0ffb4549e9 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Tue, 29 Mar 2022 13:55:45 -0700 Subject: [PATCH 01/47] adding notebook template function --- synthtool/languages/python.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/synthtool/languages/python.py b/synthtool/languages/python.py index 78a6a0ff2..d0d3ed9f2 100644 --- a/synthtool/languages/python.py +++ b/synthtool/languages/python.py @@ -51,6 +51,10 @@ SAMPLES_TEMPLATE_PATH = Path(CommonTemplates()._template_root) / "python_samples" +NOTEBOOK_TEMPLATE_PATH = ( + Path(CommonTemplates()._template_root) / "python_notebooks_testing_pipeline" +) + def fix_pb2_headers(*, proto_root: str = "**/*_pb2.py") -> None: s.replace( @@ -94,6 +98,14 @@ def _get_sample_readme_metadata(sample_dir: Path) -> dict: return sample_metadata +def python_notebooks_testing_pipeline() -> None: + in_client_library = Path("owlbot.py").exists() + if in_client_library: + excludes = [] + _tracked_paths.add(NOTEBOOK_TEMPLATE_PATH) + s.copy([NOTEBOOK_TEMPLATE_PATH], excludes=excludes) + + def py_samples(*, root: PathOrStr = None, skip_readmes: bool = False) -> None: """ Find all samples projects and render templates. From d181aa9b2be3b0a92f730c9bc6faa89adf642d80 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Fri, 1 Apr 2022 21:37:22 +0000 Subject: [PATCH 02/47] changing template path --- synthtool/languages/python.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synthtool/languages/python.py b/synthtool/languages/python.py index d0d3ed9f2..df5ef00f1 100644 --- a/synthtool/languages/python.py +++ b/synthtool/languages/python.py @@ -52,10 +52,9 @@ SAMPLES_TEMPLATE_PATH = Path(CommonTemplates()._template_root) / "python_samples" NOTEBOOK_TEMPLATE_PATH = ( - Path(CommonTemplates()._template_root) / "python_notebooks_testing_pipeline" + Path(CommonTemplates().python_notebooks) / "python_notebooks_testing_pipeline" ) - def fix_pb2_headers(*, proto_root: str = "**/*_pb2.py") -> None: s.replace( proto_root, From 57803cd70c171648696e899a00b84d566e16dbf6 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Fri, 1 Apr 2022 23:19:22 +0000 Subject: [PATCH 03/47] changing template path --- synthtool/languages/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/languages/python.py b/synthtool/languages/python.py index df5ef00f1..5a2027db2 100644 --- a/synthtool/languages/python.py +++ b/synthtool/languages/python.py @@ -52,7 +52,7 @@ SAMPLES_TEMPLATE_PATH = Path(CommonTemplates()._template_root) / "python_samples" NOTEBOOK_TEMPLATE_PATH = ( - Path(CommonTemplates().python_notebooks) / "python_notebooks_testing_pipeline" + Path(CommonTemplates().python_notebooks()) / "python_notebooks_testing_pipeline" ) def fix_pb2_headers(*, proto_root: str = "**/*_pb2.py") -> None: From 54e16db7044e2e78cae4f9d786fab6d57fef5949 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 30 Mar 2022 11:23:53 -0400 Subject: [PATCH 04/47] chore: add googleapis/yoshi-python to CODEOWNERS (#1380) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 90dce7857..a4eb21098 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,4 @@ -* @SurferJeffAtGoogle @chingor13 @bcoe @googleapis/github-automation +* @SurferJeffAtGoogle @chingor13 @bcoe @googleapis/yoshi-python @googleapis/github-automation # assign templates to the language team that owns them synthtool/gcp/templates/java_library/ @googleapis/yoshi-java From 7a9d26c3028bfe2f00bf642cb5ced62fd5624bd7 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 30 Mar 2022 11:54:29 -0400 Subject: [PATCH 05/47] chore(python): add E231 to .flake8 ignore list (#1379) --- synthtool/gcp/templates/python_library/.flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_library/.flake8 b/synthtool/gcp/templates/python_library/.flake8 index 29227d4cf..2e4387498 100644 --- a/synthtool/gcp/templates/python_library/.flake8 +++ b/synthtool/gcp/templates/python_library/.flake8 @@ -16,7 +16,7 @@ # Generated by synthtool. DO NOT EDIT! [flake8] -ignore = E203, E266, E501, W503 +ignore = E203, E231, E266, E501, W503 exclude = # Exclude generated code. **/proto/** From d02aa7220e7eb847f239977d6e44c836ce30373b Mon Sep 17 00:00:00 2001 From: losalex <90795544+losalex@users.noreply.github.com> Date: Wed, 30 Mar 2022 14:56:52 -0700 Subject: [PATCH 06/47] chore: Enable Size-Label bot in all googleapis Python notebook testing (#1384) * chore: Enable Size-Label bot in all googleapis Python notebook testing repositories Auto-label T-shirt size indicator should be assigned on every new pull request in all googleapis Python notebook testing repositories * Remove product Remove product since it is by default true --- .../python_notebooks_testing_pipeline/.github/auto-label.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/auto-label.yaml diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/auto-label.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/auto-label.yaml new file mode 100644 index 000000000..09c8d735b --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/auto-label.yaml @@ -0,0 +1,2 @@ +requestsize: + enabled: true From 55b0a55212dd5f1de3d5fd72594a40b0b3aa1aa0 Mon Sep 17 00:00:00 2001 From: losalex <90795544+losalex@users.noreply.github.com> Date: Wed, 30 Mar 2022 16:22:19 -0700 Subject: [PATCH 07/47] chore(python): Enable size-label bot (#1383) * chore: Enable Size-Label bot in all googleapis Python repositories Auto-label T-shirt size indicator should be assigned on every new pull request in all googleapis Python repositories * Remove product Remove product since it is by default true Co-authored-by: Anthonios Partheniou --- auto-label.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 auto-label.yaml diff --git a/auto-label.yaml b/auto-label.yaml new file mode 100644 index 000000000..09c8d735b --- /dev/null +++ b/auto-label.yaml @@ -0,0 +1,2 @@ +requestsize: + enabled: true From 8d8f411e7962d0884cd96766a67eec18671e8ee9 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 30 Mar 2022 19:35:57 -0400 Subject: [PATCH 08/47] chore(python): update .pre-commit-config.yaml to use black==22.3.0 (#1378) Co-authored-by: Tim Swast --- synthtool/gcp/templates/python_library/.pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_library/.pre-commit-config.yaml b/synthtool/gcp/templates/python_library/.pre-commit-config.yaml index 62eb5a77d..46d237160 100644 --- a/synthtool/gcp/templates/python_library/.pre-commit-config.yaml +++ b/synthtool/gcp/templates/python_library/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 22.3.0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 From 992b75e75c96731b0796d8deefac84738c9a009c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 31 Mar 2022 21:34:26 -0400 Subject: [PATCH 09/47] chore(python): Enable size-label bot (#1385) --- .../gcp/templates/python_library/.github/auto-label.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename auto-label.yaml => synthtool/gcp/templates/python_library/.github/auto-label.yaml (100%) diff --git a/auto-label.yaml b/synthtool/gcp/templates/python_library/.github/auto-label.yaml similarity index 100% rename from auto-label.yaml rename to synthtool/gcp/templates/python_library/.github/auto-label.yaml From ac05cf28fa83cc35addbacdb8f38fea3cde60400 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Apr 2022 04:22:56 +0200 Subject: [PATCH 10/47] chore: remove dependency on google-api-core (#1372) * chore(deps): update dependency google-api-core to v2.7.1 * chore: remove dependency on google-api-core Co-authored-by: Anthonios Partheniou --- requirements.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1ba585fb5..91072046f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,12 +8,6 @@ deprecation protobuf==3.19.3 watchdog -# https://github.com/googleapis/gapic-generator/issues/3334 -# temporarily install google-api-core for gapic-generator-python -# Bazel should install this, but there is currently a bug -# that causes the package to be skipped -google-api-core==2.0.0 - # some java processing requires xml handling lxml From f19c2cf0cb9aa58b83f706016b46e25de6736852 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Apr 2022 12:22:32 +0200 Subject: [PATCH 11/47] chore(deps): update dependency numpy to v1.22.3 (#1387) --- .../.cloud-build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt index 8f173ce00..8189df0d9 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt @@ -2,7 +2,7 @@ ipython==7.0 jupyter==1.0 nbconvert==6.0 papermill==2.3 -numpy==1.22.1 +numpy==1.22.3 pandas==1.2 matplotlib==3.4 tabulate==0.8.9 \ No newline at end of file From b139c2bfb052b7c87c3c580de8c62be0f733113d Mon Sep 17 00:00:00 2001 From: Dan Lee <71398022+dandhlee@users.noreply.github.com> Date: Fri, 1 Apr 2022 06:30:53 -0400 Subject: [PATCH 12/47] chore(deps): disable dependency dashboard (#1223) Co-authored-by: Anthonios Partheniou --- renovate.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renovate.json b/renovate.json index f45d8f110..67b45874a 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,6 @@ { "extends": [ - "config:base" + "config:base", + ":disableDependencyDashboard" ] } From 802863abacb278c5e8d62f1bfc462ff40bb1f2ed Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Apr 2022 12:43:04 +0200 Subject: [PATCH 13/47] chore(deps): update dependency pandas to v1.4.1 (#1258) Co-authored-by: Anthonios Partheniou --- .../.cloud-build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt index 8189df0d9..099a4a8e8 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt @@ -3,6 +3,6 @@ jupyter==1.0 nbconvert==6.0 papermill==2.3 numpy==1.22.3 -pandas==1.2 +pandas==1.4.1 matplotlib==3.4 tabulate==0.8.9 \ No newline at end of file From 1144bd67638cd3e2fb4b5246f6bcdc3618df6bb8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Apr 2022 06:54:27 -0400 Subject: [PATCH 14/47] build(deps): bump ipython (#1336) Bumps [ipython](https://github.com/ipython/ipython) from 7.0 to 7.16.3. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.0.0...7.16.3) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../.cloud-build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt index 099a4a8e8..9a2de96b6 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt @@ -1,4 +1,4 @@ -ipython==7.0 +ipython==7.16.3 jupyter==1.0 nbconvert==6.0 papermill==2.3 From b3b46aad0db77d43affd03779f65c96aa3e71d9c Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Fri, 1 Apr 2022 04:04:57 -0700 Subject: [PATCH 15/47] chore: remove use of googleapis-discovery (#1280) * chore: remove use of googleapis-discovery * remove constant and method for googleapis-discovery Co-authored-by: Anthonios Partheniou --- synthtool/gcp/gapic_bazel.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/synthtool/gcp/gapic_bazel.py b/synthtool/gcp/gapic_bazel.py index f2523618e..1334a8cf1 100644 --- a/synthtool/gcp/gapic_bazel.py +++ b/synthtool/gcp/gapic_bazel.py @@ -23,9 +23,6 @@ GOOGLEAPIS_URL: str = git.make_repo_clone_url("googleapis/googleapis") GOOGLEAPIS_PRIVATE_URL: str = git.make_repo_clone_url("googleapis/googleapis-private") -GOOGLEAPIS_DISCOVERY_URL: str = git.make_repo_clone_url( - "googleapis/googleapis-discovery" -) DISCOVERY_ARTIFACT_MANAGER_URL: str = git.make_repo_clone_url( "googleapis/discovery-artifact-manager" ) @@ -94,9 +91,6 @@ def _generate_code( if discogapic: api_definitions_repo = self._clone_discovery_artifact_manager() api_definitions_repo_name = "discovery-artifact-manager" - elif diregapic: - api_definitions_repo = self._clone_googleapis_discovery() - api_definitions_repo_name = "googleapis-discovery" elif private: api_definitions_repo = self._clone_googleapis_private() api_definitions_repo_name = "googleapis_private" @@ -296,22 +290,6 @@ def _clone_googleapis_private(self): return self._googleapis_private - def _clone_googleapis_discovery(self): - if self._googleapis_discovery: - return self._googleapis_discovery - - if LOCAL_GOOGLEAPIS_DISCOVERY: - self._googleapis_discovery = Path(LOCAL_GOOGLEAPIS_DISCOVERY).expanduser() - logger.debug( - f"Using local googleapis-discovery at {self._googleapis_discovery}" - ) - - else: - logger.debug("Cloning googleapis-discovery.") - self._googleapis_discovery = git.clone(GOOGLEAPIS_DISCOVERY_URL) - - return self._googleapis_discovery - def _clone_discovery_artifact_manager(self): if self._discovery_artifact_manager: return self._discovery_artifact_manager From 77b8cd3e43144e512ada066cb0e4e8a5ddafcca6 Mon Sep 17 00:00:00 2001 From: Anurag Kumar Date: Fri, 1 Apr 2022 19:56:34 +0530 Subject: [PATCH 16/47] chore: add classifiers for python 3.7, 3.8 and 3.9 (#1247) added Support for python 3.7, 3.8, 3.9 Co-authored-by: Anthonios Partheniou --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 0241ea5c8..6026f28c6 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,9 @@ 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Operating System :: OS Independent', 'Topic :: Internet', ], From 0844f13aaa334be2a3dbd96c986401afdb9c975a Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 1 Apr 2022 11:40:13 -0400 Subject: [PATCH 17/47] chore(python): refactor unit / system test dependency install (#1294) * chore(python): refactor unit / system test dependency install Closes #1185. * chore: use editable installs for local deps * chore: don't install deps using '-e' * chore: deprecate 'unit_test_external_dependencies' * fix: install standard + main unit test deps together FBO pip resolver. Co-authored-by: Anthonios Partheniou --- .../templates/python_library/noxfile.py.j2 | 153 ++++++++++++------ tests/test_python_library.py | 80 ++++++--- 2 files changed, 169 insertions(+), 64 deletions(-) diff --git a/synthtool/gcp/templates/python_library/noxfile.py.j2 b/synthtool/gcp/templates/python_library/noxfile.py.j2 index 2e5f31513..51cf9b61d 100644 --- a/synthtool/gcp/templates/python_library/noxfile.py.j2 +++ b/synthtool/gcp/templates/python_library/noxfile.py.j2 @@ -20,16 +20,62 @@ from __future__ import absolute_import import os import pathlib import shutil +import warnings import nox - BLACK_VERSION = "black==22.3.0" BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION="{{ default_python_version }}" -SYSTEM_TEST_PYTHON_VERSIONS=[{% for v in system_test_python_versions %}"{{v}}"{% if not loop.last %},{% endif %}{% endfor %}] -UNIT_TEST_PYTHON_VERSIONS=[{% for v in unit_test_python_versions %}"{{v}}"{% if not loop.last %},{% endif %}{% endfor %}] + +UNIT_TEST_PYTHON_VERSIONS=[{% for v in unit_test_python_versions %}"{{v}}"{% if not loop.last %}, {% endif %}{% endfor %}] +UNIT_TEST_STANDARD_DEPENDENCIES = [ + "mock", + "asyncmock", + "pytest", + "pytest-cov", + "pytest-asyncio", +] +UNIT_TEST_EXTERNAL_DEPENDENCIES = [{% for v in unit_test_external_dependencies %} + "{{v}}",{% endfor %} +] +UNIT_TEST_LOCAL_DEPENDENCIES = [{% for v in unit_test_local_dependencies %} + "{{v}}",{% endfor %} +] +UNIT_TEST_DEPENDENCIES = [{% for v in unit_test_dependencies %} + "{{v}}",{% endfor %} +] +UNIT_TEST_EXTRAS = [{% for v in unit_test_extras %} + "{{v}}",{% endfor %} +] +UNIT_TEST_EXTRAS_BY_PYTHON = {{ '{' }}{% if unit_test_extras_by_python %}{% for python_version, extras in unit_test_extras_by_python.items() %} + "{{python_version}}": [{% for v in extras %} + "{{v}}",{% endfor %} + ],{% endfor %}{% endif %} +} + +SYSTEM_TEST_PYTHON_VERSIONS=[{% for v in system_test_python_versions %}"{{v}}"{% if not loop.last %}, {% endif %}{% endfor %}] +SYSTEM_TEST_STANDARD_DEPENDENCIES = [ + "mock", "pytest", "google-cloud-testutils", +] +SYSTEM_TEST_EXTERNAL_DEPENDENCIES = [{% for v in system_test_external_dependencies %} + "{{v}}",{% endfor %} +] +SYSTEM_TEST_LOCAL_DEPENDENCIES = [{% for v in system_test_local_dependencies %} + "{{v}}",{% endfor %} +] +SYSTEM_TEST_DEPENDENCIES = [{% for v in system_test_dependencies %} + "{{v}}",{% endfor %} +] +SYSTEM_TEST_EXTRAS = [{% for v in system_test_extras %} + "{{v}}",{% endfor %} +] +SYSTEM_TEST_EXTRAS_BY_PYTHON = {{ '{' }}{% if system_test_extras_by_python %}{% for python_version, extras in system_test_extras_by_python.items() %} + "{{python_version}}": [{% for v in extras %} + "{{v}}",{% endfor %} + ],{% endfor %}{% endif %} +} CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() @@ -81,31 +127,41 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") +def install_unittest_dependencies(session, *constraints): + standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES + session.install(*standard_deps, *constraints) + + if UNIT_TEST_EXTERNAL_DEPENDENCIES: + warnings.warn( + "'unit_test_external_dependencies' is deprecated. Instead, please " + "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.", + DeprecationWarning, + ) + session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_LOCAL_DEPENDENCIES: + session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_EXTRAS_BY_PYTHON: + extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif UNIT_TEST_EXTRAS: + extras = UNIT_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + def default(session): # Install all test dependencies, then install this package in-place. constraints_path = str( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) - session.install("mock", "asyncmock", "pytest", "pytest-cov", "pytest-asyncio", "-c", constraints_path) - {% for d in unit_test_external_dependencies -%} - session.install("{{d}}", "-c", constraints_path) - {% endfor %} - {% for dependency in unit_test_local_dependencies %}session.install("-e", "{{dependency}}", "-c", constraints_path) - {% endfor %} - {% for dependency in unit_test_dependencies %}session.install("-e", "{{dependency}}", "-c", constraints_path){% endfor %} - {%- if unit_test_extras_by_python %} - {% for extras_python in unit_test_extras_by_python %} - {%- if not loop.first %}el{% endif %}if session.python == "{{extras_python}}": - extras = "[{{",".join(unit_test_extras_by_python[extras_python])}}]" - {% endfor %}else: - extras = "{%- if unit_test_extras %}[{{",".join(unit_test_extras)}}]{% endif %}" - session.install("-e", f".{extras}", "-c", constraints_path) - {% elif unit_test_extras %} - session.install("-e", ".[{{",".join(unit_test_extras)}}]", "-c", constraints_path) - {% else %} - session.install("-e", ".", "-c", constraints_path) - {% endif %} + install_unittest_dependencies(session, "-c", constraints_path) # Run py.test against the unit tests. session.run( @@ -128,6 +184,35 @@ def unit(session): default(session) +def install_systemtest_dependencies(session, *constraints): + + # Use pre-release gRPC for system tests. + session.install("--pre", "grpcio") + + session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTERNAL_DEPENDENCIES: + session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_LOCAL_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTRAS_BY_PYTHON: + extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif SYSTEM_TEST_EXTRAS: + extras = SYSTEM_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): """Run the system test suite.""" @@ -150,29 +235,7 @@ def system(session): if not system_test_exists and not system_test_folder_exists: session.skip("System tests were not found") - # Use pre-release gRPC for system tests. - session.install("--pre", "grpcio") - - # Install all test dependencies, then install this package into the - # virtualenv's dist-packages. - session.install("mock", "pytest", "google-cloud-testutils"{% for d in system_test_external_dependencies %}, "{{d}}"{% endfor %}, "-c", constraints_path) - - {%- if system_test_local_dependencies %} - {% for dependency in system_test_local_dependencies %}session.install("-e", "{{dependency}}", "-c", constraints_path) - {% endfor %} - {%- endif %} - {%- if system_test_extras_by_python %} - {% for extras_python in system_test_extras_by_python %} - {%- if not loop.first %}el{% endif %}if session.python == "{{extras_python}}": - extras = "[{{",".join(system_test_extras_by_python[extras_python])}}]" - {% endfor %}else: - extras = "{%- if system_test_extras %}[{{",".join(system_test_extras)}}]{% endif %}" - session.install("-e", f".{extras}", "-c", constraints_path) - {% elif system_test_extras %} - session.install("-e", ".[{{",".join(system_test_extras)}}]", "-c", constraints_path) - {% else %} - session.install("-e", ".", "-c", constraints_path) - {% endif %} + install_systemtest_dependencies(session, "-c", constraints_path) # Run py.test against the system tests. if system_test_exists: diff --git a/tests/test_python_library.py b/tests/test_python_library.py index a0a3ab160..e91a5ceaf 100644 --- a/tests/test_python_library.py +++ b/tests/test_python_library.py @@ -30,43 +30,67 @@ @pytest.mark.parametrize( ["template_kwargs", "expected_text"], [ - ({}, ["import nox", 'session.install("-e", ".", "-c", constraints_path)']), + ({}, ["import nox", 'session.install("-e", ".", *constraints)']), ( {"unit_test_local_dependencies": ["../testutils", "../unitutils"]}, [ - 'session.install("-e", "../testutils", "-c", constraints_path)', - 'session.install("-e", "../unitutils", "-c", constraints_path)', + """\ +UNIT_TEST_LOCAL_DEPENDENCIES = [ + "../testutils", + "../unitutils", +]""", ], ), ( {"system_test_local_dependencies": ["../testutils", "../sysutils"]}, [ - 'session.install("-e", "../testutils", "-c", constraints_path)', - 'session.install("-e", "../sysutils", "-c", constraints_path)', + """\ +SYSTEM_TEST_LOCAL_DEPENDENCIES = [ + "../testutils", + "../sysutils", +]""", ], ), ( {"unit_test_extras": ["abc", "def"]}, - ['session.install("-e", ".[abc,def]", "-c", constraints_path)'], + [ + """\ +UNIT_TEST_EXTRAS = [ + "abc", + "def", +]""", + ], ), ( {"system_test_extras": ["abc", "def"]}, - ['session.install("-e", ".[abc,def]", "-c", constraints_path)'], + """\ +SYSTEM_TEST_EXTRAS = [ + "abc", + "def", +]""", ), ( {"unit_test_extras_by_python": {"3.8": ["abc", "def"]}}, [ - 'if session.python == "3.8":\n extras = "[abc,def]"', - 'else:\n extras = ""', - 'session.install("-e", f".{extras}", "-c", constraints_path)', + """\ +UNIT_TEST_EXTRAS_BY_PYTHON = { + "3.8": [ + "abc", + "def", + ], +}""", ], ), ( {"system_test_extras_by_python": {"3.8": ["abc", "def"]}}, [ - 'if session.python == "3.8":\n extras = "[abc,def]"', - 'else:\n extras = ""', - 'session.install("-e", f".{extras}", "-c", constraints_path)', + """\ +SYSTEM_TEST_EXTRAS_BY_PYTHON = { + "3.8": [ + "abc", + "def", + ], +}""", ], ), ( @@ -75,9 +99,18 @@ "unit_test_extras_by_python": {"3.8": ["abc", "def"]}, }, [ - 'if session.python == "3.8":\n extras = "[abc,def]"', - 'else:\n extras = "[tuv,wxyz]"', - 'session.install("-e", f".{extras}", "-c", constraints_path)', + """\ +UNIT_TEST_EXTRAS = [ + "tuv", + "wxyz", +]""", + """\ +UNIT_TEST_EXTRAS_BY_PYTHON = { + "3.8": [ + "abc", + "def", + ], +}""", ], ), ( @@ -86,9 +119,18 @@ "system_test_extras_by_python": {"3.8": ["abc", "def"]}, }, [ - 'if session.python == "3.8":\n extras = "[abc,def]"', - 'else:\n extras = "[tuv,wxyz]"', - 'session.install("-e", f".{extras}", "-c", constraints_path)', + """\ +SYSTEM_TEST_EXTRAS = [ + "tuv", + "wxyz", +]""", + """\ +SYSTEM_TEST_EXTRAS_BY_PYTHON = { + "3.8": [ + "abc", + "def", + ], +}""", ], ), ], From bbeb6436d11d2d7ed2c4f16fad6c53411133fdb3 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Apr 2022 18:34:29 +0200 Subject: [PATCH 18/47] chore(python_notebooks): update dependency black to v22 (#1396) --- .../.github/workflows/linter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt index 271a44c80..782e29ab7 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt @@ -2,7 +2,7 @@ git+https://github.com/tensorflow/docs ipython jupyter nbconvert -black==21.8b0 +black==22.3.0 pyupgrade==2.7.3 isort==5.10.1 flake8==4.0.1 From 90c0e89f50e50f6cf4986a87ea9f5f1477b09e42 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Apr 2022 18:51:19 +0200 Subject: [PATCH 19/47] chore(deps): update dependency ipython to v8 (#1397) Co-authored-by: Anthonios Partheniou --- .../.cloud-build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt index 9a2de96b6..a31104871 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt @@ -1,4 +1,4 @@ -ipython==7.16.3 +ipython==8.2.0 jupyter==1.0 nbconvert==6.0 papermill==2.3 From 39c2d19247e6bc2dd99d39eaaaf9ff799023d5c1 Mon Sep 17 00:00:00 2001 From: losalex <90795544+losalex@users.noreply.github.com> Date: Fri, 1 Apr 2022 12:17:55 -0700 Subject: [PATCH 20/47] chore: Enable Size-Label bot in all googleapis NodeJs repositories (#1382) * chore: Enable Size-Label bot in all googleapis NodeJs repositories Auto-label T-shirt size indicator should be assigned on every new pull request in all googleapis NodeJs repositories * Remove product Remove product since it is by default true --- synthtool/gcp/templates/node_library/.github/auto-label.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 synthtool/gcp/templates/node_library/.github/auto-label.yaml diff --git a/synthtool/gcp/templates/node_library/.github/auto-label.yaml b/synthtool/gcp/templates/node_library/.github/auto-label.yaml new file mode 100644 index 000000000..09c8d735b --- /dev/null +++ b/synthtool/gcp/templates/node_library/.github/auto-label.yaml @@ -0,0 +1,2 @@ +requestsize: + enabled: true From a60843cda9aad6698ff130bdc39a4cf69dae1e9f Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Apr 2022 21:27:31 +0200 Subject: [PATCH 21/47] chore(deps): update dependency setuptools to v61 (#1398) Co-authored-by: Anthonios Partheniou --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 91072046f..a66f59ba5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools==54.2.0 +setuptools==61.3.0 nox requests From 37b20a9ec52350e6a9bb46c9dea8809698d53bdc Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Apr 2022 21:49:30 +0200 Subject: [PATCH 22/47] chore(deps): update dependency org.sonatype.plugins:nexus-staging-maven-plugin to v1.6.12 (#1388) Co-authored-by: Neenu Shaji --- synthtool/gcp/templates/java_library/samples/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/java_library/samples/pom.xml b/synthtool/gcp/templates/java_library/samples/pom.xml index 8d3c41952..0ed0e3337 100644 --- a/synthtool/gcp/templates/java_library/samples/pom.xml +++ b/synthtool/gcp/templates/java_library/samples/pom.xml @@ -46,7 +46,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.8 + 1.6.12 true From 818677ea2c42fe19a082ee9b28e6d6caf4d0250b Mon Sep 17 00:00:00 2001 From: Lo Ferris <50979514+loferris@users.noreply.github.com> Date: Fri, 1 Apr 2022 13:52:07 -0700 Subject: [PATCH 23/47] ci: update testing pipeline for notebooks (#1345) * update testing pipeline * kokoro fix * fixing linting errors * Update synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py Co-authored-by: Anthonios Partheniou * Update synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py Co-authored-by: Anthonios Partheniou * resolving license year * removing redundant print statements * adding developer todos * reverting protos * resetting protos to main Co-authored-by: Anthonios Partheniou --- .../.cloud-build/ExecuteNotebook.py | 171 ---------- .../cleanup/cleanup-cloudbuild.yaml | 2 +- .../.cloud-build/cleanup/cleanup.py | 17 +- .../cleanup/resource_cleanup_manager.py | 3 +- .../execute_changed_notebooks_cli.py | 99 ++++++ ...py => execute_changed_notebooks_helper.py} | 315 ++++++++++-------- .../.cloud-build/execute_notebook_cli.py | 40 +++ .../.cloud-build/execute_notebook_helper.py | 89 +++++ .../.cloud-build/execute_notebook_remote.py | 90 +++++ ...book-execution-test-cloudbuild-single.yaml | 31 ++ .../notebook-execution-test-cloudbuild.yaml | 39 +-- .../.cloud-build/requirements.txt | 6 +- .../.cloud-build/test_folders.txt | 1 + .../{ => utils}/NotebookProcessors.py | 7 +- .../{ => utils}/UpdateNotebookVariables.py | 3 +- .../.cloud-build/utils/__init__.py | 0 .../.cloud-build/utils/utils.py | 67 ++++ .../.github/workflows/ci.yaml | 2 + .../.github/workflows/linter/run_linter.sh | 49 ++- 19 files changed, 651 insertions(+), 380 deletions(-) delete mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteNotebook.py create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py rename synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/{ExecuteChangedNotebooks.py => execute_changed_notebooks_helper.py} (53%) create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_cli.py create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_remote.py create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild-single.yaml create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/test_folders.txt rename synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/{ => utils}/NotebookProcessors.py (92%) rename synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/{ => utils}/UpdateNotebookVariables.py (98%) create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/__init__.py create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteNotebook.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteNotebook.py deleted file mode 100644 index f9fdbb857..000000000 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteNotebook.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python -# # Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import sys -import nbformat -import os -import errno -from NotebookProcessors import RemoveNoExecuteCells, UpdateVariablesPreprocessor -from typing import Dict, Tuple -import papermill as pm -import shutil -import virtualenv -import uuid -from jupyter_client.kernelspecapp import KernelSpecManager - -# This script is used to execute a notebook and write out the output notebook. -# The replaces calling the nbconvert via command-line, which doesn't write the output notebook correctly when there are errors during execution. - -STAGING_FOLDER = "staging" -ENVIRONMENTS_PATH = "environments" -KERNELS_SPECS_PATH = "kernel_specs" - - -def create_and_install_kernel() -> Tuple[str, str]: - # Create environment - kernel_name = str(uuid.uuid4()) - env_name = f"{ENVIRONMENTS_PATH}/{kernel_name}" - # venv.create(env_name, system_site_packages=True, with_pip=True) - virtualenv.cli_run([env_name, "--system-site-packages"]) - - # Create kernel spec - kernel_spec = { - "argv": [ - f"{env_name}/bin/python", - "-m", - "ipykernel_launcher", - "-f", - "{connection_file}", - ], - "display_name": "Python 3", - "language": "python", - } - kernel_spec_folder = os.path.join(KERNELS_SPECS_PATH, kernel_name) - kernel_spec_file = os.path.join(kernel_spec_folder, "kernel.json") - - # Create kernel spec folder - if not os.path.exists(os.path.dirname(kernel_spec_file)): - try: - os.makedirs(os.path.dirname(kernel_spec_file)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - - with open(kernel_spec_file, mode="w", encoding="utf-8") as f: - json.dump(kernel_spec, f) - - # Install kernel - kernel_spec_manager = KernelSpecManager() - kernel_spec_manager.install_kernel_spec( - source_dir=kernel_spec_folder, kernel_name=kernel_name - ) - - return kernel_name, env_name - - -def execute_notebook( - notebook_file_path: str, - output_file_folder: str, - replacement_map: Dict[str, str], - should_log_output: bool, - should_use_new_kernel: bool, -): - # Create staging directory if it doesn't exist - staging_file_path = f"{STAGING_FOLDER}/{notebook_file_path}" - if not os.path.exists(os.path.dirname(staging_file_path)): - try: - os.makedirs(os.path.dirname(staging_file_path)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - - file_name = os.path.basename(os.path.normpath(notebook_file_path)) - - # Create environments folder - if not os.path.exists(ENVIRONMENTS_PATH): - try: - os.makedirs(ENVIRONMENTS_PATH) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - - # Create and install kernel - kernel_name = next( - iter(KernelSpecManager().find_kernel_specs().keys()), None - ) # Find first existing kernel and use as default - env_name = None - if should_use_new_kernel: - kernel_name, env_name = create_and_install_kernel() - - # Read notebook - with open(notebook_file_path) as f: - nb = nbformat.read(f, as_version=4) - - has_error = False - - # Execute notebook - try: - # Create preprocessors - remove_no_execute_cells_preprocessor = RemoveNoExecuteCells() - update_variables_preprocessor = UpdateVariablesPreprocessor( - replacement_map=replacement_map - ) - - # Use no-execute preprocessor - (nb, resources,) = remove_no_execute_cells_preprocessor.preprocess(nb) - - (nb, resources) = update_variables_preprocessor.preprocess(nb, resources) - - # print(f"Staging modified notebook to: {staging_file_path}") - with open(staging_file_path, mode="w", encoding="utf-8") as f: - nbformat.write(nb, f) - - # Execute notebook - pm.execute_notebook( - input_path=staging_file_path, - output_path=staging_file_path, - kernel_name=kernel_name, - progress_bar=should_log_output, - request_save_on_cell_execute=should_log_output, - log_output=should_log_output, - stdout_file=sys.stdout if should_log_output else None, - stderr_file=sys.stderr if should_log_output else None, - ) - except Exception: - # print(f"Error executing the notebook: {notebook_file_path}.\n\n") - has_error = True - - raise - - finally: - # Clear env - if env_name is not None: - shutil.rmtree(path=env_name) - - # Copy execute notebook - output_file_path = os.path.join( - output_file_folder, "failure" if has_error else "success", file_name - ) - - # Create directories if they don't exist - if not os.path.exists(os.path.dirname(output_file_path)): - try: - os.makedirs(os.path.dirname(output_file_path)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - - # print(f"Writing output to: {output_file_path}") - shutil.move(staging_file_path, output_file_path) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-cloudbuild.yaml index 890f5c4e9..8f83c9644 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-cloudbuild.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-cloudbuild.yaml @@ -5,4 +5,4 @@ steps: args: - -c - 'python3 -m pip install -U -r .cloud-build/cleanup/cleanup-requirements.txt && python3 .cloud-build/cleanup/cleanup.py' -timeout: 86400s \ No newline at end of file +timeout: 86400s diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py index 35ec9dd20..e5d345074 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# # Copyright 2021 Google LLC +# # Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from typing import List from resource_cleanup_manager import ( ResourceCleanupManager, @@ -36,15 +37,13 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo resource_name = manager.resource_name(resource) print(f"Will delete '{type_name}': {resource_name}") else: - manager.delete(resource) - + try: + manager.delete(resource) + except Exception as exception: + print(exception) -def set_dry_run(dry_run_status: bool): - if dry_run_status is True: - return True - print("Starting cleanup in dry run mode...") - return False +is_dry_run = False # List of all cleanup managers managers = [ @@ -53,4 +52,4 @@ def set_dry_run(dry_run_status: bool): ModelResourceCleanupManager(), ] -run_cleanup_managers(managers=managers, is_dry_run=set_dry_run(False)) +run_cleanup_managers(managers=managers, is_dry_run=is_dry_run) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/resource_cleanup_manager.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/resource_cleanup_manager.py index 11f45dcc4..3f4c7f344 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/resource_cleanup_manager.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/resource_cleanup_manager.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# # Copyright 2021 Google LLC +# # Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import abc from google.cloud import aiplatform from typing import Any diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py new file mode 100644 index 000000000..4092178ee --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A CLI to process changed notebooks and execute them on Google Cloud Build""" + +import argparse +import pathlib +import execute_changed_notebooks_helper + + +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ("yes", "true", "t", "y", "1"): + return True + elif v.lower() in ("no", "false", "f", "n", "0"): + return False + else: + raise argparse.ArgumentTypeError("String value expected.") + + +parser = argparse.ArgumentParser(description="Run changed notebooks.") +parser.add_argument( + "--test_paths_file", + type=pathlib.Path, + help="The path to the file that has newline-limited folders of notebooks that should be tested.", + required=True, +) +parser.add_argument( + "--base_branch", + help="The base git branch to diff against to find changed files.", + required=False, +) +parser.add_argument( + "--container_uri", + type=str, + help="The container uri to run each notebook in.", + required=True, +) +parser.add_argument( + "--variable_project_id", + type=str, + help="The GCP project id. This is used to inject a variable value into the notebook before running.", + required=True, +) +parser.add_argument( + "--variable_region", + type=str, + help="The GCP region. This is used to inject a variable value into the notebook before running.", + required=True, +) +parser.add_argument( + "--staging_bucket", + type=str, + help="The GCS bucket for staging temporary files.", + required=True, +) +parser.add_argument( + "--artifacts_bucket", + type=str, + help="The GCP directory for storing executed notebooks.", + required=True, +) +parser.add_argument( + "--should_parallelize", + type=str2bool, + nargs="?", + const=True, + default=True, + help="Should run notebooks in parallel.", +) + +args = parser.parse_args() + +notebooks = execute_changed_notebooks_helper.get_changed_notebooks( + test_paths_file=args.test_paths_file, base_branch=args.base_branch, +) + +execute_changed_notebooks_helper.process_and_execute_notebooks( + notebooks=notebooks, + container_uri=args.container_uri, + staging_bucket=args.staging_bucket, + artifacts_bucket=args.artifacts_bucket, + variable_project_id=args.variable_project_id, + variable_region=args.variable_region, + should_parallelize=args.should_parallelize, +) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py similarity index 53% rename from synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py index 53974f26a..228801744 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# # Copyright 2021 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,30 +12,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import argparse + +import concurrent import dataclasses import datetime import functools -import pathlib import os +import pathlib +import nbformat +import re import subprocess -from pathlib import Path from typing import List, Optional -import concurrent from tabulate import tabulate +import operator -import ExecuteNotebook - - -def str2bool(v): - if isinstance(v, bool): - return v - if v.lower() in ("yes", "true", "t", "y", "1"): - return True - elif v.lower() in ("no", "false", "f", "n", "0"): - return False - else: - raise argparse.ArgumentTypeError("Boolean value expected.") +import execute_notebook_remote +from utils import util, NotebookProcessors +from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata def format_timedelta(delta: datetime.timedelta) -> str: @@ -61,48 +54,127 @@ def format_timedelta(delta: datetime.timedelta) -> str: @dataclasses.dataclass class NotebookExecutionResult: - notebook: str + name: str duration: datetime.timedelta is_pass: bool + log_url: str + output_uri: str + build_id: str error_message: Optional[str] -def execute_notebook( - artifacts_path: str, +def _process_notebook( + notebook_path: str, variable_project_id: str, variable_region: str, +): + # Read notebook + with open(notebook_path) as f: + nb = nbformat.read(f, as_version=4) + + # Create preprocessors + remove_no_execute_cells_preprocessor = NotebookProcessors.RemoveNoExecuteCells() + update_variables_preprocessor = NotebookProcessors.UpdateVariablesPreprocessor( + replacement_map={"PROJECT_ID": variable_project_id, "REGION": variable_region}, + ) + + # Use no-execute preprocessor + (nb, resources,) = remove_no_execute_cells_preprocessor.preprocess(nb) + + (nb, resources) = update_variables_preprocessor.preprocess(nb, resources) + + with open(notebook_path, mode="w", encoding="utf-8") as new_file: + nbformat.write(nb, new_file) + + +def _create_tag(filepath: str) -> str: + tag = os.path.basename(os.path.normpath(filepath)) + tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag) + + if tag.startswith(".") or tag.startswith("-"): + tag = tag[1:] + + return tag + + +def process_and_execute_notebook( + container_uri: str, + staging_bucket: str, + artifacts_bucket: str, variable_project_id: str, variable_region: str, - should_log_output: bool, - should_use_new_kernel: bool, notebook: str, + should_get_tail_logs: bool = False, ) -> NotebookExecutionResult: print(f"Running notebook: {notebook}") + # Create paths + notebook_output_uri = "/".join([artifacts_bucket, pathlib.Path(notebook).name]) + + # Create tag from notebook + tag = _create_tag(filepath=notebook) + result = NotebookExecutionResult( - notebook=notebook, + name=tag, duration=datetime.timedelta(seconds=0), is_pass=False, + output_uri=notebook_output_uri, + log_url="", + build_id="", error_message=None, ) + # TODO: Handle cases where multiple notebooks have the same name time_start = datetime.datetime.now() + operation = None try: - ExecuteNotebook.execute_notebook( - notebook_file_path=notebook, - output_file_folder=artifacts_path, - replacement_map={ - "PROJECT_ID": variable_project_id, - "REGION": variable_region, - }, - should_log_output=should_log_output, - should_use_new_kernel=should_use_new_kernel, + # Pre-process notebook by substituting variable names + _process_notebook( + notebook_path=notebook, + variable_project_id=variable_project_id, + variable_region=variable_region, + ) + + # Upload the pre-processed code to a GCS bucket + code_archive_uri = util.archive_code_and_upload(staging_bucket=staging_bucket) + + operation = execute_notebook_remote.execute_notebook_remote( + code_archive_uri=code_archive_uri, + notebook_uri=notebook, + notebook_output_uri=notebook_output_uri, + container_uri=container_uri, + tag=tag, ) + + operation_metadata = BuildOperationMetadata(mapping=operation.metadata) + result.build_id = operation_metadata.build.id + result.log_url = operation_metadata.build.log_url + + # Block and wait for the result + operation.result() + result.duration = datetime.datetime.now() - time_start result.is_pass = True print(f"{notebook} PASSED in {format_timedelta(result.duration)}.") except Exception as error: + result.error_message = str(error) + + if operation and should_get_tail_logs: + # Extract the logs + logs_bucket = operation_metadata.build.logs_bucket + + # Download tail end of logs file + log_file_uri = f"{logs_bucket}/log-{result.build_id}.txt" + + # Use gcloud to get tail + try: + result.error_message = subprocess.check_output( + ["gsutil", "cat", "-r", "-1000", log_file_uri], encoding="UTF-8" + ) + except Exception as error: + result.error_message = str(error) + result.duration = datetime.datetime.now() - time_start result.is_pass = False - result.error_message = str(error) + print( f"{notebook} FAILED in {format_timedelta(result.duration)}: {result.error_message}" ) @@ -110,39 +182,12 @@ def execute_notebook( return result -def run_changed_notebooks( - test_paths_file: str, - base_branch: Optional[str], - output_folder: str, - variable_project_id: str, - variable_region: str, - should_parallelize: bool, - should_use_separate_kernels: bool, -): +def get_changed_notebooks( + test_paths_file: str, base_branch: Optional[str] = None, +) -> List[str]: """ - Run the notebooks that exist under the folders defined in the test_paths_file. - It only runs notebooks that have differences from the Git base_branch. - The executed notebooks are saved in the output_folder. - Variables are also injected into the notebooks such as the variable_project_id and variable_region. - Args: - test_paths_file (str): - Required. The new-line delimited file to folders and files that need checking. - Folders are checked recursively. - base_branch (str): - Optional. If provided, only the files that have changed from the base_branch will be checked. - If not provided, all files will be checked. - output_folder (str): - Required. The folder to write executed notebooks to. - variable_project_id (str): - Required. The value for PROJECT_ID to inject into notebooks. - variable_region (str): - Required. The value for REGION to inject into notebooks. - should_parallelize (bool): - Required. Should run notebooks in parallel using a thread pool as opposed to in sequence. - should_use_separate_kernels (bool): - Note: Dependencies don't install correctly when this is set to True - See https://github.com/nteract/papermill/issues/625 - Required. Should run each notebook in a separate and independent virtual environment. + Get the notebooks that exist under the folders defined in the test_paths_file. + It only returns notebooks that have differences from the Git base_branch. """ test_paths = [] @@ -170,14 +215,43 @@ def run_changed_notebooks( notebooks = notebooks.decode("utf-8").split("\n") notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")] notebooks = [notebook for notebook in notebooks if len(notebook) > 0] - notebooks = [notebook for notebook in notebooks if Path(notebook).exists()] + notebooks = [notebook for notebook in notebooks if pathlib.Path(notebook).exists()] + + return notebooks - # Create paths - artifacts_path = Path(output_folder) - artifacts_path.mkdir(parents=True, exist_ok=True) - artifacts_path.joinpath("success").mkdir(parents=True, exist_ok=True) - artifacts_path.joinpath("failure").mkdir(parents=True, exist_ok=True) +def process_and_execute_notebooks( + notebooks: List[str], + container_uri: str, + staging_bucket: str, + artifacts_bucket: str, + variable_project_id: str, + variable_region: str, + should_parallelize: bool, +): + """ + Run the notebooks that exist under the folders defined in the test_paths_file. + It only runs notebooks that have differences from the Git base_branch. + The executed notebooks are saved in the artifacts_bucket. + Variables are also injected into the notebooks such as the variable_project_id and variable_region. + Args: + test_paths_file (str): + Required. The new-line delimited file to folders and files that need checking. + Folders are checked recursively. + base_branch (str): + Optional. If provided, only the files that have changed from the base_branch will be checked. + If not provided, all files will be checked. + staging_bucket (str): + Required. The GCS staging bucket to write source code to. + artifacts_bucket (str): + Required. The GCS staging bucket to write executed notebooks to. + variable_project_id (str): + Required. The value for PROJECT_ID to inject into notebooks. + variable_region (str): + Required. The value for REGION to inject into notebooks. + should_parallelize (bool): + Required. Should run notebooks in parallel using a thread pool as opposed to in sequence. + """ notebook_execution_results: List[NotebookExecutionResult] = [] if len(notebooks) > 0: @@ -191,25 +265,25 @@ def run_changed_notebooks( notebook_execution_results = list( executor.map( functools.partial( - execute_notebook, - artifacts_path, + process_and_execute_notebook, + container_uri, + staging_bucket, + artifacts_bucket, variable_project_id, variable_region, - False, - should_use_separate_kernels, ), notebooks, ) ) else: notebook_execution_results = [ - execute_notebook( - artifacts_path=artifacts_path, + process_and_execute_notebook( + container_uri=container_uri, + staging_bucket=staging_bucket, + artifacts_bucket=artifacts_bucket, variable_project_id=variable_project_id, variable_region=variable_region, notebook=notebook, - should_log_output=True, - should_use_new_kernel=should_use_separate_kernels, ) for notebook in notebooks ] @@ -218,87 +292,36 @@ def run_changed_notebooks( print("\n=== RESULTS ===\n") - notebooks_sorted = sorted( + results_sorted = sorted( notebook_execution_results, key=lambda result: result.is_pass, reverse=True, ) + # Print results print( tabulate( [ [ - os.path.basename(os.path.normpath(result.notebook)), + result.name, "PASSED" if result.is_pass else "FAILED", format_timedelta(result.duration), - result.error_message or "--", + result.log_url, ] - for result in notebooks_sorted + for result in results_sorted ], - headers=["file", "status", "duration", "error"], + headers=["build_tag", "status", "duration", "log_url"], ) ) print("\n=== END RESULTS===\n") + total_notebook_duration = functools.reduce( + operator.add, + [datetime.timedelta(seconds=0)] + + [result.duration for result in results_sorted], + ) + + print(f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}") -parser = argparse.ArgumentParser(description="Run changed notebooks.") -parser.add_argument( - "--test_paths_file", - type=pathlib.Path, - help="The path to the file that has newline-limited folders of notebooks that should be tested.", - required=True, -) -parser.add_argument( - "--base_branch", - help="The base git branch to diff against to find changed files.", - required=False, -) -parser.add_argument( - "--output_folder", - type=pathlib.Path, - help="The path to the folder to store executed notebooks.", - required=True, -) -parser.add_argument( - "--variable_project_id", - type=str, - help="The GCP project id. This is used to inject a variable value into the notebook before running.", - required=True, -) -parser.add_argument( - "--variable_region", - type=str, - help="The GCP region. This is used to inject a variable value into the notebook before running.", - required=True, -) - -# Note: Dependencies don't install correctly when this is set to True -parser.add_argument( - "--should_parallelize", - type=str2bool, - nargs="?", - const=True, - default=False, - help="Should run notebooks in parallel.", -) - -# Note: This isn't guaranteed to work correctly due to existing Papermill issue -# See https://github.com/nteract/papermill/issues/625 -parser.add_argument( - "--should_use_separate_kernels", - type=str2bool, - nargs="?", - const=True, - default=False, - help="(Experimental) Should run each notebook in a separate and independent virtual environment.", -) - -args = parser.parse_args() -run_changed_notebooks( - test_paths_file=args.test_paths_file, - base_branch=args.base_branch, - output_folder=args.output_folder, - variable_project_id=args.variable_project_id, - variable_region=args.variable_region, - should_parallelize=args.should_parallelize, - should_use_separate_kernels=args.should_use_separate_kernels, -) + # Raise error if any notebooks failed + if not all([result.is_pass for result in results_sorted]): + raise RuntimeError("Notebook failures detected. See logs for details") diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_cli.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_cli.py new file mode 100644 index 000000000..9545f9c4c --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_cli.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A CLI to download (optional) and run a single notebook locally""" + +import argparse +import execute_notebook_helper + +parser = argparse.ArgumentParser(description="Run a single notebook locally.") +parser.add_argument( + "--notebook_source", + type=str, + help="Local filepath or GCS URI to notebook.", + required=True, +) +parser.add_argument( + "--output_file_or_uri", + type=str, + help="Local file or GCS URI to save executed notebook to.", + required=True, +) + +args = parser.parse_args() +execute_notebook_helper.execute_notebook( + notebook_source=args.notebook_source, + output_file_or_uri=args.output_file_or_uri, + should_log_output=True, +) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py new file mode 100644 index 000000000..32c7de372 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Methods to run a notebook locally""" + +import sys +import os +import errno +import papermill as pm +import shutil + +from utils import util +from google.cloud.aiplatform import utils + +# This script is used to execute a notebook and write out the output notebook. + + +def execute_notebook( + notebook_source: str, output_file_or_uri: str, should_log_output: bool, +): + """Execute a single notebook using Papermill""" + file_name = os.path.basename(os.path.normpath(notebook_source)) + + # Download notebook if it's a GCS URI + if notebook_source.startswith("gs://"): + # Extract uri components + bucket_name, prefix = utils.extract_bucket_and_prefix_from_gcs_path( + notebook_source + ) + + # Download remote notebook to local file system + notebook_source = file_name + util.download_file( + bucket_name=bucket_name, blob_name=prefix, destination_file=notebook_source + ) + + execution_exception = None + + # Execute notebook + try: + # Execute notebook + pm.execute_notebook( + input_path=notebook_source, + output_path=notebook_source, + progress_bar=should_log_output, + request_save_on_cell_execute=should_log_output, + log_output=should_log_output, + stdout_file=sys.stdout if should_log_output else None, + stderr_file=sys.stderr if should_log_output else None, + ) + except Exception as exception: + execution_exception = exception + finally: + # Copy executed notebook + if output_file_or_uri.startswith("gs://"): + # Upload to GCS path + util.upload_file(notebook_source, remote_file_path=output_file_or_uri) + + print("\n=== EXECUTION FINISHED ===\n") + print( + f"Please debug the executed notebook by downloading: {output_file_or_uri}" + ) + print("\n======\n") + else: + # Create directories if they don't exist + if not os.path.exists(os.path.dirname(output_file_or_uri)): + try: + os.makedirs(os.path.dirname(output_file_or_uri)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + + print(f"Writing output to: {output_file_or_uri}") + shutil.move(notebook_source, output_file_or_uri) + + if execution_exception: + raise execution_exception diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_remote.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_remote.py new file mode 100644 index 000000000..fc4bc6411 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_remote.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Methods to run a notebook on Google Cloud Build""" + +from google.protobuf import duration_pb2 +from yaml.loader import FullLoader + +import google.auth +from google.cloud.devtools import cloudbuild_v1 +from google.cloud.devtools.cloudbuild_v1.types import Source, StorageSource + +from typing import Optional +import yaml + +from google.cloud.aiplatform import utils +from google.api_core import operation + +CLOUD_BUILD_FILEPATH = ".cloud-build/notebook-execution-test-cloudbuild-single.yaml" +TIMEOUT_IN_SECONDS = 86400 + + +def execute_notebook_remote( + code_archive_uri: str, + notebook_uri: str, + notebook_output_uri: str, + container_uri: str, + tag: Optional[str], +) -> operation.Operation: + """Create and execute a single notebook on Google Cloud Build""" + + # Authorize the client with Google defaults + credentials, project_id = google.auth.default() + client = cloudbuild_v1.services.cloud_build.CloudBuildClient() + + build = cloudbuild_v1.Build() + + # The following build steps will output "hello world" + # For more information on build configuration, see + # https://cloud.google.com/build/docs/configuring-builds/create-basic-configuration + cloudbuild_config = yaml.load(open(CLOUD_BUILD_FILEPATH), Loader=FullLoader) + + substitutions = { + "_PYTHON_IMAGE": container_uri, + "_NOTEBOOK_GCS_URI": notebook_uri, + "_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri, + } + + ( + source_archived_file_gcs_bucket, + source_archived_file_gcs_object, + ) = utils.extract_bucket_and_prefix_from_gcs_path(code_archive_uri) + + build.source = Source( + storage_source=StorageSource( + bucket=source_archived_file_gcs_bucket, + object_=source_archived_file_gcs_object, + ) + ) + + build.steps = cloudbuild_config["steps"] + build.substitutions = substitutions + build.timeout = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS) + build.queue_ttl = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS) + + if tag: + build.tags = [tag] + + operation = client.create_build(project_id=project_id, build=build) + # Print the in-progress operation + # TODO(developer): Uncomment next two lines + # print("IN PROGRESS:") + # print(operation.metadata) + + # Print the completed status + # TODO(developer): Uncomment next line + # print("RESULT:", result.status) + return operation diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild-single.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild-single.yaml new file mode 100644 index 000000000..5e12755b1 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild-single.yaml @@ -0,0 +1,31 @@ +steps: + # Show the gcloud info and check if gcloud exists + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'gcloud config list' + # Check the Python version + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'python3 .cloud-build/CheckPythonVersion.py' + # Install Python dependencies + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt' + # Install Python dependencies and run testing script + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"' + env: + - 'IS_TESTING=1' +timeout: 86400s +options: + pool: + name: ${_PRIVATE_POOL_NAME} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml index 15d1d8283..4d9d84756 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -1,44 +1,41 @@ steps: # Show the gcloud info and check if gcloud exists - - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' + - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - -c - 'gcloud config list' + # # Clone the Git repo + # - name: ${_PYTHON_IMAGE} + # entrypoint: git + # args: ['clone', "${_GIT_REPO}", "--branch", "${_GIT_BRANCH_NAME}", "."] # Check the Python version - - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' + - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - -c - 'python3 .cloud-build/CheckPythonVersion.py' # Fetch base branch if required - - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' + - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - -c - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi' # Install Python dependencies - - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' - entrypoint: pip - args: ['install', '--upgrade', '--user', '--requirement', '.cloud-build/requirements.txt'] + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt' # Install Python dependencies and run testing script - - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' + - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - -c - - 'python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}' + - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET} --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}' env: - 'IS_TESTING=1' - # Manually copy artifacts to GCS - - name: gcr.io/cloud-builders/gsutil - entrypoint: /bin/sh - args: - - -c - - 'if [ $(ls -pR "/workspace/${BUILD_ID}" | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil -m -q rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi' - # Fail if there is anything in the failure folder - - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' - entrypoint: /bin/sh - args: - - -c - - 'echo "Download executed notebooks with this command: \"mkdir -p artifacts && gsutil rsync -r gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} artifacts/\"" && if [ "$(ls -A /workspace/${BUILD_ID}/failure | wc -l)" -ne 0 ]; then exit 1; else exit 0; fi' -timeout: 86400s \ No newline at end of file +timeout: 86400s +options: + pool: + name: ${_PRIVATE_POOL_NAME} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt index a31104871..c111762a1 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt @@ -5,4 +5,8 @@ papermill==2.3 numpy==1.22.3 pandas==1.4.1 matplotlib==3.4 -tabulate==0.8.9 \ No newline at end of file +tabulate==0.8.9 +google-cloud-aiplatform +google-cloud-storage +google-cloud-build +gcloud \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/test_folders.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/test_folders.txt new file mode 100644 index 000000000..6eb63c85f --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/test_folders.txt @@ -0,0 +1 @@ +notebooks/official \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/NotebookProcessors.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py similarity index 92% rename from synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/NotebookProcessors.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py index 90a61a51c..1b7934ea6 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/NotebookProcessors.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# # Copyright 2021 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from nbconvert.preprocessors import Preprocessor from typing import Dict -import UpdateNotebookVariables +from . import UpdateNotebookVariables as update_notebook_variables class RemoveNoExecuteCells(Preprocessor): @@ -40,7 +41,7 @@ def update_variables(content: str, replacement_map: Dict[str, str]): # VARIABLE_NAME = '[description]' for variable_name, variable_value in replacement_map.items(): - content = UpdateNotebookVariables.get_updated_value( + content = update_notebook_variables.get_updated_value( content=content, variable_name=variable_name, variable_value=variable_value, diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/UpdateNotebookVariables.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/UpdateNotebookVariables.py similarity index 98% rename from synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/UpdateNotebookVariables.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/UpdateNotebookVariables.py index b357d7854..c602e4903 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/UpdateNotebookVariables.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/UpdateNotebookVariables.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# # Copyright 2021 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import re """ diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/__init__.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py new file mode 100644 index 000000000..c72aedd1d --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import subprocess +import tarfile +import uuid + + +def download_file(bucket_name: str, blob_name: str, destination_file: str) -> str: + """Copies a remote GCS file to a local path""" + remote_file_path = "".join(["gs://", "/".join([bucket_name, blob_name])]) + + subprocess.check_output( + ["gsutil", "cp", remote_file_path, destination_file], encoding="UTF-8" + ) + + return destination_file + + +def upload_file(local_file_path: str, remote_file_path: str,) -> str: + """Copies a local file to a GCS path""" + subprocess.check_output( + ["gsutil", "cp", local_file_path, remote_file_path], encoding="UTF-8" + ) + + return remote_file_path + + +def archive_code_and_upload(staging_bucket: str): + # Archive all source in current directory + unique_id = uuid.uuid4() + source_archived_file = f"source_archived_{unique_id}.tar.gz" + + git_files = subprocess.check_output( + ["git", "ls-tree", "-r", "HEAD", "--name-only"], encoding="UTF-8" + ).split("\n") + + with tarfile.open(source_archived_file, "w:gz") as tar: + for file in git_files: + if len(file) > 0 and os.path.exists(file): + tar.add(file) + + # Upload archive to GCS bucket + source_archived_file_gcs = upload_file( + local_file_path=f"{source_archived_file}", + remote_file_path="/".join( + [staging_bucket, "code_archives", source_archived_file] + ), + ) + + print(f"Uploaded source code archive to {source_archived_file_gcs}") + + return source_archived_file_gcs diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml index d8b78f268..7fab63af1 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml @@ -19,8 +19,10 @@ jobs: - name: Format and lint notebooks run: | set +e + .github/workflows/linter/run_linter.sh -t RTN=$? + if [ "$RTN" != "0" ]; then echo "There were problems formatting/linting the notebooks." echo "Please run the following commands locally from the root directory to attempt to autofix the issues:" diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh index 7b7cadb22..102c33a51 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script automatically formats and lints all notebooks that have changed from the head of the master branch. +# This script automatically formats and lints all notebooks that have changed from the head of the main branch. # # Options: # -t: Test-mode. Only test if format and linting are required but make no changes to files. @@ -32,16 +32,16 @@ RTN="0" is_test=false -# Process all options supplied on the command line +# Process all options supplied on the command line while getopts 'tc' arg; do case $arg in - 't') - is_test=true - ;; - *) - echo "Unimplemented flag" - exit 1 - ;; + 't') + is_test=true + ;; + *) + echo "Unimplemented flag" + exit 1 + ;; esac done @@ -50,15 +50,13 @@ echo "Test mode: $is_test" # Only check notebooks in test folders modified in this pull request. # Note: Use process substitution to persist the data in the array notebooks=() -while read -r file || [ -n "$line" ]; -do +while read -r file || [ -n "$line" ]; do notebooks+=("$file") done < <(git diff --name-only main... | grep '\.ipynb$') problematic_notebooks=() if [ ${#notebooks[@]} -gt 0 ]; then - for notebook in "${notebooks[@]}" - do + for notebook in "${notebooks[@]}"; do if [ -f "$notebook" ]; then echo "Checking notebook: ${notebook}" @@ -68,7 +66,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then ISORT_RTN="0" FLAKE8_RTN="0" - if [ "$is_test" = true ] ; then + if [ "$is_test" = true ]; then echo "Running nbfmt..." python3 -m tensorflow_docs.tools.nbfmt --remove_outputs --test "$notebook" NBFMT_RTN=$? @@ -82,24 +80,24 @@ if [ ${#notebooks[@]} -gt 0 ]; then python3 -m nbqa isort "$notebook" --check ISORT_RTN=$? echo "Running flake8..." - python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,W291,E203,E999,E111,E113 + python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291 FLAKE8_RTN=$? else echo "Running black..." - python3 -m black "$notebook" - BLACK_RTN=$? + python3 -m nbqa black "$notebook" + BLACK_RTN=$? echo "Running pyupgrade..." - python3 -m nbqa pyupgrade "$notebook" --nbqa-mutate + python3 -m nbqa pyupgrade "$notebook" PYUPGRADE_RTN=$? echo "Running isort..." - python3 -m nbqa isort "$notebook" --nbqa-mutate + python3 -m nbqa isort "$notebook" ISORT_RTN=$? echo "Running nbfmt..." python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook" NBFMT_RTN=$? echo "Running flake8..." - python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,W291,E203,E999,E111,E113 - FLAKE8_RTN=$? + python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291 + FLAKE8_RTN=$? fi NOTEBOOK_RTN="0" @@ -108,7 +106,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then NOTEBOOK_RTN="$NBFMT_RTN" printf "nbfmt: Failed\n" fi - + if [ "$BLACK_RTN" != "0" ]; then NOTEBOOK_RTN="$BLACK_RTN" printf "black: Failed\n" @@ -131,10 +129,9 @@ if [ ${#notebooks[@]} -gt 0 ]; then echo "Notebook lint finished with return code = $NOTEBOOK_RTN" echo "" - if [ "$NOTEBOOK_RTN" != "0" ] - then - problematic_notebooks+=("$notebook") - RTN=$NOTEBOOK_RTN + if [ "$NOTEBOOK_RTN" != "0" ]; then + problematic_notebooks+=("$notebook") + RTN=$NOTEBOOK_RTN fi fi done From 4a5191b205977c69d8c2251290544b435338a44b Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Apr 2022 23:16:17 +0200 Subject: [PATCH 24/47] chore(deps): update actions/setup-python action to v3 (#1395) Co-authored-by: Anthonios Partheniou --- .../python_notebooks_testing_pipeline/.github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml index 7fab63af1..3d52b25ff 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 - name: Fetch pull request branch uses: actions/checkout@v2 with: From 007266457d12dfbf8c6f17bf60e93c00c924f5f1 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 1 Apr 2022 18:13:51 -0400 Subject: [PATCH 25/47] chore: upgrade black in noxfile.py to 22.3.0 (#1377) * chore: upgrade black to 22.3.0 * update lint also * run blacken session * ci: use click>8.0 for blacken/lint sessions * chore: fix typo Co-authored-by: nicain * rebase error Co-authored-by: nicain --- noxfile.py | 7 ++++--- synthtool/gcp/common.py | 2 +- synthtool/gcp/samples.py | 2 +- .../execute_changed_notebooks_cli.py | 3 ++- .../execute_changed_notebooks_helper.py | 16 ++++++++++++---- .../.cloud-build/execute_notebook_helper.py | 4 +++- .../.cloud-build/utils/NotebookProcessors.py | 3 ++- .../.cloud-build/utils/utils.py | 5 ++++- synthtool/languages/java.py | 3 ++- synthtool/languages/php.py | 17 +++++++---------- synthtool/languages/python.py | 18 +++++++++--------- synthtool/protos/metadata_pb2.py | 4 +++- synthtool/protos/preconfig_pb2.py | 4 +++- synthtool/sources/git.py | 5 ++++- synthtool/transforms.py | 7 +++++-- tests/test_php.py | 6 ++---- tests/test_python_library.py | 5 ++++- 17 files changed, 68 insertions(+), 43 deletions(-) diff --git a/noxfile.py b/noxfile.py index 7de717838..16243f8c1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -20,16 +20,17 @@ def generate_protos(session): session.run( "python", "-m", "grpc_tools.protoc", "-Isynthtool/protos", "--python_out=synthtool/protos", "synthtool/protos/metadata.proto", "synthtool/protos/preconfig.proto") -@nox.session(python=['3.6', '3.8']) +@nox.session(python=['3.6', '3.9']) def blacken(session): - session.install('black==19.10b0') + session.install('black==22.3.0', 'click>8.0') session.run('black', 'synthtool', 'tests') @nox.session(python=['3.6', '3.8', '3.9']) def lint(session): - session.install('mypy==0.790', 'flake8', 'black==19.10b0') + session.install('mypy==0.790', 'flake8', 'black==22.3.0') session.run('pip', 'install', '-e', '.') + session.run('pip', 'install', 'click>8.0') session.run('black', '--check', 'synthtool', 'tests') session.run('flake8', 'synthtool', 'tests') session.run('mypy', 'synthtool') diff --git a/synthtool/gcp/common.py b/synthtool/gcp/common.py index 0a79d544d..b1fbb6564 100644 --- a/synthtool/gcp/common.py +++ b/synthtool/gcp/common.py @@ -423,7 +423,7 @@ def detect_versions( def decamelize(value: str): - """ parser to convert fooBar.js to Foo Bar. """ + """Parser to convert fooBar.js to Foo Bar.""" if not value: return "" str_decamelize = re.sub("^.", value[0].upper(), value) # apple -> Apple. diff --git a/synthtool/gcp/samples.py b/synthtool/gcp/samples.py index 35b9f214f..21a18d98b 100644 --- a/synthtool/gcp/samples.py +++ b/synthtool/gcp/samples.py @@ -78,7 +78,7 @@ def all_samples(sample_globs: List[str]) -> List[Dict[str, str]]: def _decamelize(value: str): - """ parser to convert fooBar.js to Foo Bar. """ + """Parser to convert fooBar.js to Foo Bar.""" if not value: return "" str_decamelize = re.sub("^.", value[0].upper(), value) # apple -> Apple. diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py index 4092178ee..84d1de387 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py @@ -85,7 +85,8 @@ def str2bool(v): args = parser.parse_args() notebooks = execute_changed_notebooks_helper.get_changed_notebooks( - test_paths_file=args.test_paths_file, base_branch=args.base_branch, + test_paths_file=args.test_paths_file, + base_branch=args.base_branch, ) execute_changed_notebooks_helper.process_and_execute_notebooks( diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py index 228801744..f454205f7 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py @@ -64,7 +64,9 @@ class NotebookExecutionResult: def _process_notebook( - notebook_path: str, variable_project_id: str, variable_region: str, + notebook_path: str, + variable_project_id: str, + variable_region: str, ): # Read notebook with open(notebook_path) as f: @@ -77,7 +79,10 @@ def _process_notebook( ) # Use no-execute preprocessor - (nb, resources,) = remove_no_execute_cells_preprocessor.preprocess(nb) + ( + nb, + resources, + ) = remove_no_execute_cells_preprocessor.preprocess(nb) (nb, resources) = update_variables_preprocessor.preprocess(nb, resources) @@ -183,7 +188,8 @@ def process_and_execute_notebook( def get_changed_notebooks( - test_paths_file: str, base_branch: Optional[str] = None, + test_paths_file: str, + base_branch: Optional[str] = None, ) -> List[str]: """ Get the notebooks that exist under the folders defined in the test_paths_file. @@ -293,7 +299,9 @@ def process_and_execute_notebooks( print("\n=== RESULTS ===\n") results_sorted = sorted( - notebook_execution_results, key=lambda result: result.is_pass, reverse=True, + notebook_execution_results, + key=lambda result: result.is_pass, + reverse=True, ) # Print results diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py index 32c7de372..d59b7b616 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py @@ -28,7 +28,9 @@ def execute_notebook( - notebook_source: str, output_file_or_uri: str, should_log_output: bool, + notebook_source: str, + output_file_or_uri: str, + should_log_output: bool, ): """Execute a single notebook using Papermill""" file_name = os.path.basename(os.path.normpath(notebook_source)) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py index 1b7934ea6..d445750e0 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py @@ -54,7 +54,8 @@ def preprocess(self, notebook, resources=None): for cell in notebook.cells: if cell.cell_type == "code": cell.source = self.update_variables( - content=cell.source, replacement_map=self._replacement_map, + content=cell.source, + replacement_map=self._replacement_map, ) executable_cells.append(cell) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py index c72aedd1d..b4978e8b4 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py @@ -31,7 +31,10 @@ def download_file(bucket_name: str, blob_name: str, destination_file: str) -> st return destination_file -def upload_file(local_file_path: str, remote_file_path: str,) -> str: +def upload_file( + local_file_path: str, + remote_file_path: str, +) -> str: """Copies a local file to a GCS path""" subprocess.check_output( ["gsutil", "cp", local_file_path, remote_file_path], encoding="UTF-8" diff --git a/synthtool/languages/java.py b/synthtool/languages/java.py index d775467e5..2202c69e8 100644 --- a/synthtool/languages/java.py +++ b/synthtool/languages/java.py @@ -455,7 +455,8 @@ def _common_template_metadata() -> Dict[str, Any]: ) metadata["latest_bom_version"] = latest_maven_version( - group_id="com.google.cloud", artifact_id="libraries-bom", + group_id="com.google.cloud", + artifact_id="libraries-bom", ) metadata["samples"] = samples.all_samples(["samples/**/src/main/java/**/*.java"]) diff --git a/synthtool/languages/php.py b/synthtool/languages/php.py index 99310914c..2e3a5ad68 100644 --- a/synthtool/languages/php.py +++ b/synthtool/languages/php.py @@ -66,8 +66,7 @@ def _merge(src: str, dest: str, path: Path): def _find_copy_target(src: Path, version_string: str) -> typing.Optional[Path]: - """Returns a directory contains the version subdirectory. - """ + """Returns a directory contains the version subdirectory.""" logger.debug("_find_copy_target called with %s and %s", src, version_string) entries = os.scandir(src) if not entries: @@ -81,10 +80,11 @@ def _find_copy_target(src: Path, version_string: str) -> typing.Optional[Path]: def owlbot_copy_version( - src: Path, dest: Path, copy_excludes: typing.Optional[typing.List[str]] = None, + src: Path, + dest: Path, + copy_excludes: typing.Optional[typing.List[str]] = None, ) -> None: - """Copies files from a version subdirectory. - """ + """Copies files from a version subdirectory.""" logger.debug("owlbot_copy_version called from %s to %s", src, dest) if copy_excludes is None: @@ -146,8 +146,7 @@ def owlbot_main( copy_excludes: typing.Optional[typing.List[str]] = None, patch_func: typing.Callable[[], None] = owlbot_patch, ) -> None: - """Copies files from generated tree. - """ + """Copies files from generated tree.""" entries = os.scandir(src) if not entries: logger.info("there is no version subdirectory to copy") @@ -161,9 +160,7 @@ def owlbot_main( def owlbot_entrypoint(staging_dir: str = STAGING_DIR) -> None: - """Copies files from staging and template directories into current working dir. - - """ + """Copies files from staging and template directories into current working dir.""" logging.basicConfig(level=logging.INFO) logger.debug("owlbot_main called") diff --git a/synthtool/languages/python.py b/synthtool/languages/python.py index 5a2027db2..39a8ded9d 100644 --- a/synthtool/languages/python.py +++ b/synthtool/languages/python.py @@ -59,7 +59,7 @@ def fix_pb2_headers(*, proto_root: str = "**/*_pb2.py") -> None: s.replace( proto_root, PB2_HEADER, - fr"\g<1>{LICENSE}\n\n\g<2>", # change order to avoid stacking replacements + rf"\g<1>{LICENSE}\n\n\g<2>", # change order to avoid stacking replacements flags=re.DOTALL | re.MULTILINE, ) @@ -68,7 +68,7 @@ def fix_pb2_grpc_headers(*, proto_root: str = "**/*_pb2_grpc.py") -> None: s.replace( proto_root, PB2_GRPC_HEADER, - fr"{LICENSE}\n\n\g<1>\n\n\g<2>", # add line breaks to avoid stacking replacements + rf"{LICENSE}\n\n\g<1>\n\n\g<2>", # add line breaks to avoid stacking replacements ) @@ -161,15 +161,15 @@ def py_samples(*, root: PathOrStr = None, skip_readmes: bool = False) -> None: def configure_previous_major_version_branches() -> None: """Configure releases from previous major version branches by editing - `.github/release-please.yml`. + `.github/release-please.yml`. - The current library version is obtained from `version.py` in `google/**/version.py`, - or the `setup.py`. + The current library version is obtained from `version.py` in `google/**/version.py`, + or the `setup.py`. - Releases are configured for all previous major versions. For example, - if the library version is currently 3.5.1, the release-please config - will include v2, v1, and v0. - """ + Releases are configured for all previous major versions. For example, + if the library version is currently 3.5.1, the release-please config + will include v2, v1, and v0. + """ if list(Path(".").glob("google/**/version.py")): version_file = list(Path(".").glob("google/**/version.py"))[0] else: diff --git a/synthtool/protos/metadata_pb2.py b/synthtool/protos/metadata_pb2.py index 767a1ce98..fcfeb548e 100644 --- a/synthtool/protos/metadata_pb2.py +++ b/synthtool/protos/metadata_pb2.py @@ -22,7 +22,9 @@ serialized_options=None, create_key=_descriptor._internal_create_key, serialized_pb=b'\n\x0emetadata.proto\x12\x14yoshi.synth.metadata\x1a\x1fgoogle/protobuf/timestamp.proto"\xf6\x01\n\x08Metadata\x12\x33\n\x0bupdate_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x02\x18\x01\x12-\n\x07sources\x18\x02 \x03(\x0b\x32\x1c.yoshi.synth.metadata.Source\x12\x37\n\x0c\x64\x65stinations\x18\x03 \x03(\x0b\x32!.yoshi.synth.metadata.Destination\x12\x34\n\tnew_files\x18\x04 \x03(\x0b\x32\x1d.yoshi.synth.metadata.NewFileB\x02\x18\x01\x12\x17\n\x0fgenerated_files\x18\x05 \x03(\t"\xb8\x01\n\x06Source\x12.\n\x03git\x18\x01 \x01(\x0b\x32\x1f.yoshi.synth.metadata.GitSourceH\x00\x12:\n\tgenerator\x18\x02 \x01(\x0b\x32%.yoshi.synth.metadata.GeneratorSourceH\x00\x12\x38\n\x08template\x18\x03 \x01(\x0b\x32$.yoshi.synth.metadata.TemplateSourceH\x00\x42\x08\n\x06source"m\n\tGitSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06remote\x18\x02 \x01(\t\x12\x0b\n\x03sha\x18\x03 \x01(\t\x12\x14\n\x0cinternal_ref\x18\x04 \x01(\t\x12\x12\n\nlocal_path\x18\x05 \x01(\t\x12\x0b\n\x03log\x18\x06 \x01(\t"F\n\x0fGeneratorSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x14\n\x0c\x64ocker_image\x18\x03 \x01(\t"?\n\x0eTemplateSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t"\x94\x01\n\x0b\x44\x65stination\x12\x39\n\x06\x63lient\x18\x01 \x01(\x0b\x32\'.yoshi.synth.metadata.ClientDestinationH\x00\x12;\n\x07\x66ileset\x18\x02 \x01(\x0b\x32(.yoshi.synth.metadata.FileSetDestinationH\x00\x42\r\n\x0b\x44\x65stination"\x17\n\x07NewFile\x12\x0c\n\x04path\x18\x01 \x01(\t"\x7f\n\x11\x43lientDestination\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x10\n\x08\x61pi_name\x18\x02 \x01(\t\x12\x13\n\x0b\x61pi_version\x18\x03 \x01(\t\x12\x10\n\x08language\x18\x04 \x01(\t\x12\x11\n\tgenerator\x18\x05 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x06 \x01(\t"3\n\x12\x46ileSetDestination\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x03(\tb\x06proto3', - dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,], + dependencies=[ + google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR, + ], ) diff --git a/synthtool/protos/preconfig_pb2.py b/synthtool/protos/preconfig_pb2.py index 4225ea050..48a4b10d3 100644 --- a/synthtool/protos/preconfig_pb2.py +++ b/synthtool/protos/preconfig_pb2.py @@ -110,7 +110,9 @@ ), ], extensions=[], - nested_types=[_PRECONFIG_PRECLONEDREPOSENTRY,], + nested_types=[ + _PRECONFIG_PRECLONEDREPOSENTRY, + ], enum_types=[], serialized_options=None, is_extendable=False, diff --git a/synthtool/sources/git.py b/synthtool/sources/git.py index c9406a894..14db9a06a 100644 --- a/synthtool/sources/git.py +++ b/synthtool/sources/git.py @@ -72,7 +72,10 @@ def _local_default_branch(path: pathlib.Path) -> Union[str, None]: def clone( - url: str, dest: pathlib.Path = None, committish: str = None, force: bool = False, + url: str, + dest: pathlib.Path = None, + committish: str = None, + force: bool = False, ) -> pathlib.Path: """Clones a remote git repo. diff --git a/synthtool/transforms.py b/synthtool/transforms.py index 5c3170a75..c3c9dd81a 100644 --- a/synthtool/transforms.py +++ b/synthtool/transforms.py @@ -137,7 +137,8 @@ def _copy_dir_to_existing_dir( _merge_file(source_path, dest_path, merge) except Exception: logger.exception( - "_merge_file failed for %s, fall back to copy", source_path, + "_merge_file failed for %s, fall back to copy", + source_path, ) shutil.copy2(str(source_path), str(dest_path)) else: @@ -147,7 +148,9 @@ def _copy_dir_to_existing_dir( return copied -def dont_overwrite(patterns: ListOfPathsOrStrs,) -> Callable[[str, str, Path], str]: +def dont_overwrite( + patterns: ListOfPathsOrStrs, +) -> Callable[[str, str, Path], str]: """Returns a merge function that doesn't overwrite the specified files. Pass the return value to move() or copy() to avoid overwriting existing diff --git a/tests/test_php.py b/tests/test_php.py index 280383753..d3b384374 100644 --- a/tests/test_php.py +++ b/tests/test_php.py @@ -51,8 +51,7 @@ def docker_image(): @pytest.fixture(scope="session") def hybrid_tmp_path(): - """A tmp dir implementation both for local run and on Kokoro. - """ + """A tmp dir implementation both for local run and on Kokoro.""" # Trampoline mount KOKORO_ROOT at the same path. # So we can mount files under there with docker in docker. hybrid_dir = os.environ.get("KOKORO_ROOT", None) @@ -65,8 +64,7 @@ def hybrid_tmp_path(): @pytest.fixture(scope="function", params=["php_asset"]) def copy_fixture(request, hybrid_tmp_path): - """A fixture for preparing test data. - """ + """A fixture for preparing test data.""" param = request.param test_dir = Path(f"{hybrid_tmp_path}/{param}") diff --git a/tests/test_python_library.py b/tests/test_python_library.py index e91a5ceaf..de17c8be0 100644 --- a/tests/test_python_library.py +++ b/tests/test_python_library.py @@ -137,7 +137,10 @@ ) def test_library_noxfile(template_kwargs, expected_text): t = templates.Templates(PYTHON_LIBRARY) - result = t.render("noxfile.py.j2", **template_kwargs,).read_text() + result = t.render( + "noxfile.py.j2", + **template_kwargs, + ).read_text() # Validate Python syntax. result_code = compile(result, "noxfile.py", "exec") assert result_code is not None From e4f8beefdce7caa7f82ccf3467ca05e502dc1579 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 2 Apr 2022 18:46:47 +0200 Subject: [PATCH 26/47] chore(deps): update dependency pandas to v1.4.2 (#1400) --- .../.cloud-build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt index c111762a1..1d6c70f40 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt @@ -3,7 +3,7 @@ jupyter==1.0 nbconvert==6.0 papermill==2.3 numpy==1.22.3 -pandas==1.4.1 +pandas==1.4.2 matplotlib==3.4 tabulate==0.8.9 google-cloud-aiplatform From a3166c63621b9dd350d87165fc8905096e8c306a Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 2 Apr 2022 18:55:17 +0200 Subject: [PATCH 27/47] chore(deps): update dependency setuptools to v61.3.1 (#1399) Co-authored-by: Anthonios Partheniou --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a66f59ba5..d8d5200b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools==61.3.0 +setuptools==61.3.1 nox requests From 196270c858b4aef290a54dc8094d3668a3bddfb7 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 2 Apr 2022 20:03:12 +0200 Subject: [PATCH 28/47] chore(deps): update dependency nbqa to v1.3.1 (#1391) Co-authored-by: Anthonios Partheniou --- .../.github/workflows/linter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt index 782e29ab7..34a926d50 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt @@ -6,4 +6,4 @@ black==22.3.0 pyupgrade==2.7.3 isort==5.10.1 flake8==4.0.1 -nbqa==1.2.3 \ No newline at end of file +nbqa==1.3.1 \ No newline at end of file From ba8b7b4f458ddd03433028c021ee166c50fcb4d1 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 2 Apr 2022 20:22:24 +0200 Subject: [PATCH 29/47] chore(deps): update dependency protobuf to v3.19.4 (#1389) Co-authored-by: Anthonios Partheniou --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d8d5200b3..307ad3f5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ requests pyyaml jinja2 deprecation -protobuf==3.19.3 +protobuf==3.19.4 watchdog # some java processing requires xml handling From 0638c04422e1e3995c01a50be5c2e08e515865ec Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 2 Apr 2022 20:28:31 +0200 Subject: [PATCH 30/47] chore(deps): update dependency protobuf to v3.20.0 (#1403) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 307ad3f5e..4515876c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ requests pyyaml jinja2 deprecation -protobuf==3.19.4 +protobuf==3.20.0 watchdog # some java processing requires xml handling From 505e97f93d4d09dce812e02dc89c83a14270542b Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 4 Apr 2022 16:15:02 +0200 Subject: [PATCH 31/47] chore(deps): update dependency setuptools to v62 (#1406) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4515876c6..db4e5663e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools==61.3.1 +setuptools==62.0.0 nox requests From 99951ef1ab8e5f71773a8a0e2cdc92833cd02e2b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 4 Apr 2022 13:36:09 -0400 Subject: [PATCH 32/47] chore(python): Use python 3.10.4 base image for post processor (#1405) --- docker/owlbot/python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/owlbot/python/Dockerfile b/docker/owlbot/python/Dockerfile index 432405272..4501fc86d 100644 --- a/docker/owlbot/python/Dockerfile +++ b/docker/owlbot/python/Dockerfile @@ -16,7 +16,7 @@ # build from the root of this repo: # docker build -t gcr.io/repo-automation-bots/owlbot-python -f docker/owlbot/python/Dockerfile . -FROM python:3.9.5-buster +FROM python:3.10.4-buster WORKDIR / From b9b3390dbbdb6be36bd734ecfe512532bf84c0f8 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 6 Apr 2022 00:40:45 +0200 Subject: [PATCH 33/47] chore(deps): update actions/setup-node action to v3 (#1393) Co-authored-by: Jeffrey Rennie --- .../gcp/templates/node_library/.github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml b/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml index 5d448d70f..ab8f1e386 100644 --- a/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml +++ b/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: node: [10, 12, 14, 16] steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: node-version: ${{ '{{' }} matrix.node {{ '}}' }} - run: node --version @@ -30,7 +30,7 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: node-version: 14 - run: npm install @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: node-version: 14 - run: npm install @@ -50,7 +50,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: node-version: 14 - run: npm install From d8c122560cb850bda63138c2d5b1d339f0ff7e4e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 6 Apr 2022 06:22:33 -0400 Subject: [PATCH 34/47] chore(python): add license header to auto-label.yaml (#1404) --- .../python_library/.github/auto-label.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/synthtool/gcp/templates/python_library/.github/auto-label.yaml b/synthtool/gcp/templates/python_library/.github/auto-label.yaml index 09c8d735b..41bff0b53 100644 --- a/synthtool/gcp/templates/python_library/.github/auto-label.yaml +++ b/synthtool/gcp/templates/python_library/.github/auto-label.yaml @@ -1,2 +1,15 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. requestsize: enabled: true From a0a2279c23fd951123a8a8057e8b3c8c886b75c0 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Wed, 6 Apr 2022 05:07:40 -0600 Subject: [PATCH 35/47] chore: improve logic to configure release-please for previous major versions (#1408) * fix: allow version.py files without the library version * chore: remove obsolete file * chore: run black Co-authored-by: Anthonios Partheniou --- blargh.yml | 7 ------- synthtool/languages/python.py | 22 ++++++++++++---------- 2 files changed, 12 insertions(+), 17 deletions(-) delete mode 100644 blargh.yml diff --git a/blargh.yml b/blargh.yml deleted file mode 100644 index e5f1aa652..000000000 --- a/blargh.yml +++ /dev/null @@ -1,7 +0,0 @@ -bumpMinorPreMajor: true -handleGHRelease: true -releaseType: java-yoshi -branches: - - handleGHRelease: true - releaseType: python - branch: java7 \ No newline at end of file diff --git a/synthtool/languages/python.py b/synthtool/languages/python.py index 39a8ded9d..f3b5bf38a 100644 --- a/synthtool/languages/python.py +++ b/synthtool/languages/python.py @@ -170,25 +170,27 @@ def configure_previous_major_version_branches() -> None: if the library version is currently 3.5.1, the release-please config will include v2, v1, and v0. """ - if list(Path(".").glob("google/**/version.py")): - version_file = list(Path(".").glob("google/**/version.py"))[0] - else: - version_file = Path("setup.py") # In version.py: __version__ = "1.5.2" # In setup.py: version = "1.5.2" VERSION_REGEX = ( r"(?:__)?version(?:__)?\s*=\s*[\"'](?P\d)\.[\d\.]+[\"']" ) + version_paths = list(Path(".").glob("google/**/version.py")) + [Path("setup.py")] - match = re.search(VERSION_REGEX, Path(version_file).read_text()) + major_version = None - if match is not None: - major_version = int(match.group("major_version")) - else: + for p in version_paths: + match = re.search(VERSION_REGEX, Path(p).read_text()) + + if match is not None: + major_version = int(match.group("major_version")) + break + + if major_version is None: raise RuntimeError( - "Unable to find library version in {} with regex {}".format( - version_file, VERSION_REGEX + "Unable to find library version in files {} with regex {}".format( + version_paths, VERSION_REGEX ) ) From 13722ad361ca6dcfbf4594e0028e16a550ac0c7d Mon Sep 17 00:00:00 2001 From: losalex <90795544+losalex@users.noreply.github.com> Date: Wed, 6 Apr 2022 09:28:47 -0700 Subject: [PATCH 36/47] chore: Enable Size-Label bot in all googleapis Java repositories (#1381) * chore: Enable Size-Label bot in all googleapis Java repositories Auto-label T-shirt size indicator should be assigned on every new pull request in all googleapis Java repositories * Remove product Remove product since it is by default true * add license header Co-authored-by: Neenu Shaji --- .../java_library/.github/auto-label.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 synthtool/gcp/templates/java_library/.github/auto-label.yaml diff --git a/synthtool/gcp/templates/java_library/.github/auto-label.yaml b/synthtool/gcp/templates/java_library/.github/auto-label.yaml new file mode 100644 index 000000000..4caef688b --- /dev/null +++ b/synthtool/gcp/templates/java_library/.github/auto-label.yaml @@ -0,0 +1,15 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +requestsize: + enabled: true From 4cd812bf59fefed263c3e3e36c5dbbe0c5e4c9fd Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 6 Apr 2022 20:34:32 +0200 Subject: [PATCH 37/47] chore(deps): update actions/checkout action to v3 (#1392) Co-authored-by: Jeffrey Rennie --- .../gcp/templates/node_library/.github/workflows/ci.yaml | 8 ++++---- .../.github/workflows/ci.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml b/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml index ab8f1e386..c4eb9b183 100644 --- a/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml +++ b/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml @@ -11,7 +11,7 @@ jobs: matrix: node: [10, 12, 14, 16] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: node-version: ${{ '{{' }} matrix.node {{ '}}' }} @@ -29,7 +29,7 @@ jobs: windows: runs-on: windows-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: node-version: 14 @@ -40,7 +40,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: node-version: 14 @@ -49,7 +49,7 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: node-version: 14 diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml index 3d52b25ff..2997b1c1c 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml @@ -9,7 +9,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 - name: Fetch pull request branch - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Fetch base main branch From dfa4da09c211ed5e2be4e8b31123de2f80517cfd Mon Sep 17 00:00:00 2001 From: Jeffrey Rennie Date: Wed, 6 Apr 2022 16:03:30 -0700 Subject: [PATCH 38/47] chore: run tests with Python 3.10 (#1407) Co-authored-by: Anthonios Partheniou --- noxfile.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/noxfile.py b/noxfile.py index 16243f8c1..1b0a024ff 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,19 +14,19 @@ import nox -@nox.session(python=['3.6']) +@nox.session(python=['3.6', '3.10']) def generate_protos(session): session.install("grpcio-tools") session.run( "python", "-m", "grpc_tools.protoc", "-Isynthtool/protos", "--python_out=synthtool/protos", "synthtool/protos/metadata.proto", "synthtool/protos/preconfig.proto") -@nox.session(python=['3.6', '3.9']) +@nox.session(python=['3.6', '3.10']) def blacken(session): session.install('black==22.3.0', 'click>8.0') session.run('black', 'synthtool', 'tests') -@nox.session(python=['3.6', '3.8', '3.9']) +@nox.session(python=['3.6', '3.10']) def lint(session): session.install('mypy==0.790', 'flake8', 'black==22.3.0') session.run('pip', 'install', '-e', '.') @@ -36,7 +36,7 @@ def lint(session): session.run('mypy', 'synthtool') -@nox.session(python=['3.6', '3.8']) +@nox.session(python=['3.6', '3.10']) def test(session): session.install('pytest', 'pytest-cov', 'requests_mock', 'watchdog', 'flake8') session.run('pip', 'install', '-e', '.') From fd8b41834a83e6baf94ff67711a05f1954716259 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 7 Apr 2022 01:52:19 +0200 Subject: [PATCH 39/47] chore(deps): update dependency python to v3.10.4 (#1386) Co-authored-by: Anthonios Partheniou Co-authored-by: Jeffrey Rennie --- docker/owlbot/nodejs/Dockerfile | 2 +- docker/owlbot/php/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/owlbot/nodejs/Dockerfile b/docker/owlbot/nodejs/Dockerfile index e1cf07aaa..e6dc7b194 100644 --- a/docker/owlbot/nodejs/Dockerfile +++ b/docker/owlbot/nodejs/Dockerfile @@ -16,7 +16,7 @@ # build from the root of this repo: # docker build -t gcr.io/repo-automation-bots/owlbot-nodejs -f docker/owlbot/nodejs/Dockerfile . -FROM python:3.9.5-buster +FROM python:3.10.4-buster WORKDIR / diff --git a/docker/owlbot/php/Dockerfile b/docker/owlbot/php/Dockerfile index fa9dd0c14..d5034b8c2 100644 --- a/docker/owlbot/php/Dockerfile +++ b/docker/owlbot/php/Dockerfile @@ -16,7 +16,7 @@ # build from the root of this repo: # docker build -t gcr.io/repo-automation-bots/owlbot-php -f docker/owlbot/php/Dockerfile . -FROM python:3.9.5-buster +FROM python:3.10.4-buster WORKDIR / From 75b8c33b1691b354c1182bae670685b29ed4ad54 Mon Sep 17 00:00:00 2001 From: Summer Ji Date: Thu, 7 Apr 2022 09:46:14 -0700 Subject: [PATCH 40/47] fix: clarify the gax-nodejs usage in README (#1352) Co-authored-by: Benjamin E. Coe Co-authored-by: Jeffrey Rennie --- synthtool/gcp/templates/node_library/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synthtool/gcp/templates/node_library/README.md b/synthtool/gcp/templates/node_library/README.md index e5a6fa5f2..f8e20c1d1 100644 --- a/synthtool/gcp/templates/node_library/README.md +++ b/synthtool/gcp/templates/node_library/README.md @@ -72,6 +72,9 @@ Google APIs Client Libraries, in [Client Libraries Explained][explained]. {% endif %}{% if 'partials' in metadata and metadata['partials']['body'] %}{{ metadata['partials']['body'] }}{% endif %} {% if metadata['samples']|length %} + +This library is built on top of google-gax as its transport layer. To customize the common behavior of this client library (timeout, retry, etc), see [Client Library Howto](https://github.com/googleapis/gax-nodejs/blob/main/client-libraries.md). + ## Samples Samples are in the [`samples/`](https://github.com/{{ metadata['repo']['repo'] }}/tree/{{ metadata['repo']['default_branch'] }}/samples) directory. Each sample's `README.md` has instructions for running its sample. From 212123b08ce17a1ca6298452b537ec04b8710aa0 Mon Sep 17 00:00:00 2001 From: "Benjamin E. Coe" Date: Thu, 7 Apr 2022 11:09:49 -0700 Subject: [PATCH 41/47] Revert "fix: clarify the gax-nodejs usage in README (#1352)" (#1409) This reverts commit e1557e468fd986c952ba718d9ff90e1d87390209. --- synthtool/gcp/templates/node_library/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/synthtool/gcp/templates/node_library/README.md b/synthtool/gcp/templates/node_library/README.md index f8e20c1d1..e5a6fa5f2 100644 --- a/synthtool/gcp/templates/node_library/README.md +++ b/synthtool/gcp/templates/node_library/README.md @@ -72,9 +72,6 @@ Google APIs Client Libraries, in [Client Libraries Explained][explained]. {% endif %}{% if 'partials' in metadata and metadata['partials']['body'] %}{{ metadata['partials']['body'] }}{% endif %} {% if metadata['samples']|length %} - -This library is built on top of google-gax as its transport layer. To customize the common behavior of this client library (timeout, retry, etc), see [Client Library Howto](https://github.com/googleapis/gax-nodejs/blob/main/client-libraries.md). - ## Samples Samples are in the [`samples/`](https://github.com/{{ metadata['repo']['repo'] }}/tree/{{ metadata['repo']['default_branch'] }}/samples) directory. Each sample's `README.md` has instructions for running its sample. From 3c349d207afa898112ee62991e6ed4b971bcee24 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sun, 10 Apr 2022 02:07:09 +0200 Subject: [PATCH 42/47] chore(deps): update dependency pyupgrade to v2.32.0 (#1412) --- .../.github/workflows/linter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt index 34a926d50..2d7591ce9 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt @@ -3,7 +3,7 @@ ipython jupyter nbconvert black==22.3.0 -pyupgrade==2.7.3 +pyupgrade==2.32.0 isort==5.10.1 flake8==4.0.1 nbqa==1.3.1 \ No newline at end of file From dbc1d4cdb6fffe1c2beaee5e2b2fbb6c6b9a7b56 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 14 Apr 2022 12:43:00 +0200 Subject: [PATCH 43/47] chore(deps): update dependency nbconvert to v6.5.0 (#1414) --- .../.cloud-build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt index 1d6c70f40..c64f61587 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt @@ -1,6 +1,6 @@ ipython==8.2.0 jupyter==1.0 -nbconvert==6.0 +nbconvert==6.5.0 papermill==2.3 numpy==1.22.3 pandas==1.4.2 From 434798fbd4bc461a2f92e804b2e6627533c6d8f0 Mon Sep 17 00:00:00 2001 From: sofisl <55454395+sofisl@users.noreply.github.com> Date: Thu, 14 Apr 2022 10:35:02 -0700 Subject: [PATCH 44/47] build: make ci testing conditional on engines field in package.json, move configs to Node 12 (#1418) * build: make ci testing conditional on engines field in package.json, move configs to Node 12 Co-authored-by: Benjamin E. Coe --- .../node_library/.github/workflows/ci.yaml | 2 +- .../templates/node_library/.kokoro/common.cfg | 2 +- .../.kokoro/continuous/node10/common.cfg | 34 --------- .../.kokoro/continuous/node10/docs.cfg | 4 -- .../.kokoro/continuous/node10/test.cfg | 9 --- .../.kokoro/presubmit/node10/common.cfg | 34 --------- .../node_library/.kokoro/release/docs.cfg | 2 +- .../node_library/.kokoro/samples-test.sh | 2 +- .../node_library/.kokoro/system-test.sh | 2 +- .../templates/node_library/.kokoro/test.sh | 2 +- synthtool/languages/node.py | 3 +- .../node_templates/standard/package.json | 72 +++++++++++++++++++ tests/test_node.py | 6 ++ 13 files changed, 86 insertions(+), 88 deletions(-) delete mode 100644 synthtool/gcp/templates/node_library/.kokoro/continuous/node10/common.cfg delete mode 100644 synthtool/gcp/templates/node_library/.kokoro/continuous/node10/docs.cfg delete mode 100644 synthtool/gcp/templates/node_library/.kokoro/continuous/node10/test.cfg delete mode 100644 synthtool/gcp/templates/node_library/.kokoro/presubmit/node10/common.cfg create mode 100644 tests/fixtures/node_templates/standard/package.json diff --git a/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml b/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml index c4eb9b183..8730b393c 100644 --- a/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml +++ b/synthtool/gcp/templates/node_library/.github/workflows/ci.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node: [10, 12, 14, 16] + node: [{{metadata['engine'] | int}}, {{metadata['engine'] | int+2}}, {{metadata['engine'] | int+4}}] steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 diff --git a/synthtool/gcp/templates/node_library/.kokoro/common.cfg b/synthtool/gcp/templates/node_library/.kokoro/common.cfg index 5072f652e..0df7d844b 100644 --- a/synthtool/gcp/templates/node_library/.kokoro/common.cfg +++ b/synthtool/gcp/templates/node_library/.kokoro/common.cfg @@ -16,7 +16,7 @@ build_file: "{{ metadata['repository_name'] }}/.kokoro/trampoline_v2.sh" # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:10-user" + value: "gcr.io/cloud-devrel-kokoro-resources/node:12-user" } env_vars: { key: "TRAMPOLINE_BUILD_FILE" diff --git a/synthtool/gcp/templates/node_library/.kokoro/continuous/node10/common.cfg b/synthtool/gcp/templates/node_library/.kokoro/continuous/node10/common.cfg deleted file mode 100644 index a27b03619..000000000 --- a/synthtool/gcp/templates/node_library/.kokoro/continuous/node10/common.cfg +++ /dev/null @@ -1,34 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Bring in codecov.io token into the build as $KOKORO_KEYSTORE_DIR/73713_dpebot_codecov_token -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "dpebot_codecov_token" - } - } -} - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "{{ metadata['repository_name'] }}/.kokoro/trampoline_v2.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:10-user" -} -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/{{ metadata['repository_name'] }}/.kokoro/test.sh" -} diff --git a/synthtool/gcp/templates/node_library/.kokoro/continuous/node10/docs.cfg b/synthtool/gcp/templates/node_library/.kokoro/continuous/node10/docs.cfg deleted file mode 100644 index 2c5d393f3..000000000 --- a/synthtool/gcp/templates/node_library/.kokoro/continuous/node10/docs.cfg +++ /dev/null @@ -1,4 +0,0 @@ -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/{{ metadata['repository_name'] }}/.kokoro/docs.sh" -} diff --git a/synthtool/gcp/templates/node_library/.kokoro/continuous/node10/test.cfg b/synthtool/gcp/templates/node_library/.kokoro/continuous/node10/test.cfg deleted file mode 100644 index 609c0cf0a..000000000 --- a/synthtool/gcp/templates/node_library/.kokoro/continuous/node10/test.cfg +++ /dev/null @@ -1,9 +0,0 @@ -# Bring in codecov.io token into the build as $KOKORO_KEYSTORE_DIR/73713_dpebot_codecov_token -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "dpebot_codecov_token" - } - } -} diff --git a/synthtool/gcp/templates/node_library/.kokoro/presubmit/node10/common.cfg b/synthtool/gcp/templates/node_library/.kokoro/presubmit/node10/common.cfg deleted file mode 100644 index a27b03619..000000000 --- a/synthtool/gcp/templates/node_library/.kokoro/presubmit/node10/common.cfg +++ /dev/null @@ -1,34 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Bring in codecov.io token into the build as $KOKORO_KEYSTORE_DIR/73713_dpebot_codecov_token -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "dpebot_codecov_token" - } - } -} - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "{{ metadata['repository_name'] }}/.kokoro/trampoline_v2.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:10-user" -} -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/{{ metadata['repository_name'] }}/.kokoro/test.sh" -} diff --git a/synthtool/gcp/templates/node_library/.kokoro/release/docs.cfg b/synthtool/gcp/templates/node_library/.kokoro/release/docs.cfg index 4c2a59419..93f7acbbf 100644 --- a/synthtool/gcp/templates/node_library/.kokoro/release/docs.cfg +++ b/synthtool/gcp/templates/node_library/.kokoro/release/docs.cfg @@ -11,7 +11,7 @@ before_action { # doc publications use a Python image. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:10-user" + value: "gcr.io/cloud-devrel-kokoro-resources/node:12-user" } # Download trampoline resources. diff --git a/synthtool/gcp/templates/node_library/.kokoro/samples-test.sh b/synthtool/gcp/templates/node_library/.kokoro/samples-test.sh index 3611785bc..91c1c7878 100755 --- a/synthtool/gcp/templates/node_library/.kokoro/samples-test.sh +++ b/synthtool/gcp/templates/node_library/.kokoro/samples-test.sh @@ -56,7 +56,7 @@ fi # codecov combines coverage across integration and unit tests. Include # the logic below for any environment you wish to collect coverage for: -COVERAGE_NODE=10 +COVERAGE_NODE=12 if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then NYC_BIN=./node_modules/nyc/bin/nyc.js if [ -f "$NYC_BIN" ]; then diff --git a/synthtool/gcp/templates/node_library/.kokoro/system-test.sh b/synthtool/gcp/templates/node_library/.kokoro/system-test.sh index d1e020ce0..b93793e4f 100755 --- a/synthtool/gcp/templates/node_library/.kokoro/system-test.sh +++ b/synthtool/gcp/templates/node_library/.kokoro/system-test.sh @@ -49,7 +49,7 @@ npm run system-test # codecov combines coverage across integration and unit tests. Include # the logic below for any environment you wish to collect coverage for: -COVERAGE_NODE=10 +COVERAGE_NODE=12 if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then NYC_BIN=./node_modules/nyc/bin/nyc.js if [ -f "$NYC_BIN" ]; then diff --git a/synthtool/gcp/templates/node_library/.kokoro/test.sh b/synthtool/gcp/templates/node_library/.kokoro/test.sh index af1ce7e33..a5c7ac04c 100755 --- a/synthtool/gcp/templates/node_library/.kokoro/test.sh +++ b/synthtool/gcp/templates/node_library/.kokoro/test.sh @@ -39,7 +39,7 @@ npm test # codecov combines coverage across integration and unit tests. Include # the logic below for any environment you wish to collect coverage for: -COVERAGE_NODE=10 +COVERAGE_NODE=12 if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then NYC_BIN=./node_modules/nyc/bin/nyc.js if [ -f "$NYC_BIN" ]; then diff --git a/synthtool/languages/node.py b/synthtool/languages/node.py index 315a4ccc5..281c9d4cd 100644 --- a/synthtool/languages/node.py +++ b/synthtool/languages/node.py @@ -24,7 +24,7 @@ import logging import shutil -_REQUIRED_FIELDS = ["name", "repository"] +_REQUIRED_FIELDS = ["name", "repository", "engines"] _TOOLS_DIRECTORY = "/synthtool" @@ -48,6 +48,7 @@ def read_metadata(): data["repository"] = f'{repo["owner"]}/{repo["name"]}' data["repository_name"] = repo["name"] data["lib_install_cmd"] = f'npm install {data["name"]}' + data["engine"] = re.search(r"([0-9][0-9])", data["engines"]["node"]).group() return data diff --git a/tests/fixtures/node_templates/standard/package.json b/tests/fixtures/node_templates/standard/package.json new file mode 100644 index 000000000..bc6436e8c --- /dev/null +++ b/tests/fixtures/node_templates/standard/package.json @@ -0,0 +1,72 @@ +{ + "name": "@google-cloud/dlp", + "description": "DLP API client for Node.js", + "version": "3.1.0", + "license": "Apache-2.0", + "author": "Google Inc", + "engines": { + "node": ">=10.0.0" + }, + "repository": "googleapis/nodejs-dlp", + "main": "build/src/index.js", + "files": [ + "build/protos", + "build/src" + ], + "keywords": [ + "google apis client", + "google api client", + "google apis", + "google api", + "google", + "google cloud platform", + "google cloud", + "cloud", + "google dlp", + "dlp", + "DLP API" + ], + "scripts": { + "test": "c8 mocha build/test", + "samples-test": "cd samples/ && npm link ../ && npm install && npm test && cd ../", + "system-test": "mocha build/system-test", + "docs": "jsdoc -c .jsdoc.js", + "lint": "gts check", + "fix": "gts fix", + "docs-test": "linkinator docs", + "clean": "gts clean", + "compile": "tsc -p . && cp -r protos build/", + "compile-protos": "compileProtos src", + "predocs-test": "npm run docs", + "prepare": "npm run compile-protos && npm run compile", + "prelint": "cd samples; npm link ../; npm install", + "precompile": "gts clean", + "api-extractor": "api-extractor run --local", + "api-documenter": "api-documenter yaml --input-folder=temp" + }, + "dependencies": { + "google-gax": "^2.9.2", + "protobufjs": "^6.8.0" + }, + "devDependencies": { + "@types/mocha": "^8.0.0", + "@types/node": "^12.0.0", + "@types/sinon": "^9.0.0", + "c8": "^7.0.0", + "gts": "^2.0.0", + "jsdoc": "^3.5.5", + "jsdoc-fresh": "^1.0.1", + "jsdoc-region-tag": "^1.0.2", + "linkinator": "^2.0.0", + "mocha": "^8.0.0", + "null-loader": "^4.0.0", + "pack-n-play": "^1.0.0-2", + "sinon": "^10.0.0", + "ts-loader": "^8.0.0", + "typescript": "^3.8.3", + "webpack": "^5.0.0", + "webpack-cli": "^4.0.0", + "@microsoft/api-documenter": "^7.8.10", + "@microsoft/api-extractor": "^7.8.10" + } + } \ No newline at end of file diff --git a/tests/test_node.py b/tests/test_node.py index a44dbd136..383876444 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -43,6 +43,12 @@ def test_quickstart_metadata_with_snippet(): assert "samples/quickstart.js" in sample_names +def test_metadata_engines_field(): + with util.chdir(FIXTURES / "node_templates" / "standard"): + metadata = node.template_metadata() + assert "10" in metadata["engine"] + + def test_quickstart_metadata_without_snippet(): with util.chdir(FIXTURES / "node_templates" / "no_quickstart_snippet"): metadata = node.template_metadata() From 905661cc406a9b9b19ecd782323e83957ba4f755 Mon Sep 17 00:00:00 2001 From: sofisl <55454395+sofisl@users.noreply.github.com> Date: Thu, 14 Apr 2022 12:55:05 -0700 Subject: [PATCH 45/47] build: sdd srs yaml file (#1419) * build: add sync-repo-settings and change branch protection --- .../.github/sync-repo-settings.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 synthtool/gcp/templates/node_library/.github/sync-repo-settings.yaml diff --git a/synthtool/gcp/templates/node_library/.github/sync-repo-settings.yaml b/synthtool/gcp/templates/node_library/.github/sync-repo-settings.yaml new file mode 100644 index 000000000..bfd74430b --- /dev/null +++ b/synthtool/gcp/templates/node_library/.github/sync-repo-settings.yaml @@ -0,0 +1,17 @@ +branchProtectionRules: + - pattern: main + isAdminEnforced: true + requiredApprovingReviewCount: 1 + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: false + requiredStatusCheckContexts: + - "ci/kokoro: Samples test" + - "ci/kokoro: System test" + - docs + - lint + - test ({{metadata['engine'] | int}}) + - test ({{metadata['engine'] | int+2}}) + - test ({{metadata['engine'] | int+4}}) + - cla/google + - windows + - OwlBot Post Processor From 59f49e7e13f6ed8b11b870378d6ed4f3e431a00f Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Tue, 19 Apr 2022 17:18:53 +0000 Subject: [PATCH 46/47] update names for template --- synthtool/languages/python.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synthtool/languages/python.py b/synthtool/languages/python.py index f3b5bf38a..f44cd3eb4 100644 --- a/synthtool/languages/python.py +++ b/synthtool/languages/python.py @@ -51,9 +51,7 @@ SAMPLES_TEMPLATE_PATH = Path(CommonTemplates()._template_root) / "python_samples" -NOTEBOOK_TEMPLATE_PATH = ( - Path(CommonTemplates().python_notebooks()) / "python_notebooks_testing_pipeline" -) +NOTEBOOK_TEMPLATE_PATH = Path(CommonTemplates()._template_root) / "python_notebooks_testing_pipeline" def fix_pb2_headers(*, proto_root: str = "**/*_pb2.py") -> None: s.replace( From 069e1a923a1e86ca2ab773ea957f272fa3b04362 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Tue, 19 Apr 2022 13:46:08 -0700 Subject: [PATCH 47/47] adding type annotation --- synthtool/languages/python.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synthtool/languages/python.py b/synthtool/languages/python.py index f44cd3eb4..1d69cc2fc 100644 --- a/synthtool/languages/python.py +++ b/synthtool/languages/python.py @@ -51,7 +51,10 @@ SAMPLES_TEMPLATE_PATH = Path(CommonTemplates()._template_root) / "python_samples" -NOTEBOOK_TEMPLATE_PATH = Path(CommonTemplates()._template_root) / "python_notebooks_testing_pipeline" +NOTEBOOK_TEMPLATE_PATH = ( + Path(CommonTemplates()._template_root) / "python_notebooks_testing_pipeline" +) + def fix_pb2_headers(*, proto_root: str = "**/*_pb2.py") -> None: s.replace( @@ -98,7 +101,7 @@ def _get_sample_readme_metadata(sample_dir: Path) -> dict: def python_notebooks_testing_pipeline() -> None: in_client_library = Path("owlbot.py").exists() if in_client_library: - excludes = [] + excludes: List[str] = [] _tracked_paths.add(NOTEBOOK_TEMPLATE_PATH) s.copy([NOTEBOOK_TEMPLATE_PATH], excludes=excludes)