From 61c9f631c549ac3ba7752499f9a16e590a81fe7c Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 3 Oct 2016 20:09:15 -0700 Subject: [PATCH 1/2] Moving package requirements into requirements.txt files. Using boilerplate code snippet in each setup.py to load the requirements.txt and adding the requirements.txt file to MANIFEST.in file. --- bigquery/MANIFEST.in | 1 + bigquery/requirements.txt | 1 + bigquery/setup.py | 9 ++++++--- bigtable/MANIFEST.in | 1 + bigtable/requirements.txt | 2 ++ bigtable/setup.py | 10 ++++++---- core/MANIFEST.in | 1 + core/requirements.txt | 5 +++++ core/setup.py | 13 ++++++------- datastore/MANIFEST.in | 1 + datastore/requirements.txt | 2 ++ datastore/setup.py | 10 ++++++---- dns/MANIFEST.in | 1 + dns/requirements.txt | 1 + dns/setup.py | 9 ++++++--- error_reporting/MANIFEST.in | 1 + error_reporting/requirements.txt | 2 ++ error_reporting/setup.py | 10 ++++++---- language/MANIFEST.in | 1 + language/requirements.txt | 1 + language/setup.py | 9 ++++++--- logging/MANIFEST.in | 1 + logging/requirements.txt | 5 +++++ logging/setup.py | 13 ++++++------- monitoring/MANIFEST.in | 1 + monitoring/requirements.txt | 1 + monitoring/setup.py | 9 ++++++--- pubsub/MANIFEST.in | 1 + pubsub/requirements.txt | 5 +++++ pubsub/setup.py | 13 ++++++------- resource_manager/MANIFEST.in | 1 + resource_manager/requirements.txt | 1 + resource_manager/setup.py | 9 ++++++--- speech/MANIFEST.in | 1 + speech/requirements.txt | 1 + speech/setup.py | 9 ++++++--- storage/MANIFEST.in | 1 + storage/requirements.txt | 1 + storage/setup.py | 9 ++++++--- translate/MANIFEST.in | 1 + translate/requirements.txt | 1 + translate/setup.py | 9 ++++++--- vision/MANIFEST.in | 1 + vision/requirements.txt | 1 + vision/setup.py | 9 ++++++--- 45 files changed, 135 insertions(+), 60 deletions(-) create mode 100644 bigquery/requirements.txt create mode 100644 bigtable/requirements.txt create mode 100644 core/requirements.txt create mode 100644 datastore/requirements.txt create mode 100644 dns/requirements.txt create mode 100644 error_reporting/requirements.txt create mode 100644 language/requirements.txt create mode 100644 logging/requirements.txt create mode 100644 monitoring/requirements.txt create mode 100644 pubsub/requirements.txt create mode 100644 resource_manager/requirements.txt create mode 100644 speech/requirements.txt create mode 100644 storage/requirements.txt create mode 100644 translate/requirements.txt create mode 100644 vision/requirements.txt diff --git a/bigquery/MANIFEST.in b/bigquery/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/bigquery/MANIFEST.in +++ b/bigquery/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/bigquery/requirements.txt b/bigquery/requirements.txt new file mode 100644 index 000000000000..db922a2bfe81 --- /dev/null +++ b/bigquery/requirements.txt @@ -0,0 +1 @@ +google-cloud-core >= 0.20.0 diff --git a/bigquery/setup.py b/bigquery/setup.py index a54bfd655f48..ae7be3dcd140 100644 --- a/bigquery/setup.py +++ b/bigquery/setup.py @@ -49,9 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-bigquery', diff --git a/bigtable/MANIFEST.in b/bigtable/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/bigtable/MANIFEST.in +++ b/bigtable/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/bigtable/requirements.txt b/bigtable/requirements.txt new file mode 100644 index 000000000000..4ca419435c7b --- /dev/null +++ b/bigtable/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-core >= 0.20.0 +grpcio >= 1.0.0, < 2.0dev diff --git a/bigtable/setup.py b/bigtable/setup.py index 6c36042e9484..ffe56ddd8916 100644 --- a/bigtable/setup.py +++ b/bigtable/setup.py @@ -49,10 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', - 'grpcio >= 1.0.0, < 2.0dev', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-bigtable', diff --git a/core/MANIFEST.in b/core/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/core/MANIFEST.in +++ b/core/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/core/requirements.txt b/core/requirements.txt new file mode 100644 index 000000000000..3d19dd813e17 --- /dev/null +++ b/core/requirements.txt @@ -0,0 +1,5 @@ +httplib2 >= 0.9.1 +googleapis-common-protos >= 1.3.4 +oauth2client >= 3.0.0, < 4.0.0dev +protobuf >= 3.0.0 +six diff --git a/core/setup.py b/core/setup.py index b59d4f5d2b3c..b554e13609cd 100644 --- a/core/setup.py +++ b/core/setup.py @@ -49,13 +49,12 @@ } -REQUIREMENTS = [ - 'httplib2 >= 0.9.1', - 'googleapis-common-protos >= 1.3.4', - 'oauth2client >= 3.0.0, < 4.0.0dev', - 'protobuf >= 3.0.0', - 'six', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-core', diff --git a/datastore/MANIFEST.in b/datastore/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/datastore/MANIFEST.in +++ b/datastore/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/datastore/requirements.txt b/datastore/requirements.txt new file mode 100644 index 000000000000..4ca419435c7b --- /dev/null +++ b/datastore/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-core >= 0.20.0 +grpcio >= 1.0.0, < 2.0dev diff --git a/datastore/setup.py b/datastore/setup.py index 17052a272044..4e9482f4cec2 100644 --- a/datastore/setup.py +++ b/datastore/setup.py @@ -49,10 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', - 'grpcio >= 1.0.0, < 2.0dev', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-datastore', diff --git a/dns/MANIFEST.in b/dns/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/dns/MANIFEST.in +++ b/dns/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/dns/requirements.txt b/dns/requirements.txt new file mode 100644 index 000000000000..db922a2bfe81 --- /dev/null +++ b/dns/requirements.txt @@ -0,0 +1 @@ +google-cloud-core >= 0.20.0 diff --git a/dns/setup.py b/dns/setup.py index ca6107dae1a8..696eae059a39 100644 --- a/dns/setup.py +++ b/dns/setup.py @@ -49,9 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-dns', diff --git a/error_reporting/MANIFEST.in b/error_reporting/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/error_reporting/MANIFEST.in +++ b/error_reporting/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/error_reporting/requirements.txt b/error_reporting/requirements.txt new file mode 100644 index 000000000000..918446385412 --- /dev/null +++ b/error_reporting/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-core >= 0.20.0 +google-cloud-logging >= 0.20.0 diff --git a/error_reporting/setup.py b/error_reporting/setup.py index 6b98eb55cf19..494285e0af6e 100644 --- a/error_reporting/setup.py +++ b/error_reporting/setup.py @@ -49,10 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', - 'google-cloud-logging >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-error-reporting', diff --git a/language/MANIFEST.in b/language/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/language/MANIFEST.in +++ b/language/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/language/requirements.txt b/language/requirements.txt new file mode 100644 index 000000000000..db922a2bfe81 --- /dev/null +++ b/language/requirements.txt @@ -0,0 +1 @@ +google-cloud-core >= 0.20.0 diff --git a/language/setup.py b/language/setup.py index 7e3907123362..9c66c0a02221 100644 --- a/language/setup.py +++ b/language/setup.py @@ -49,9 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-language', diff --git a/logging/MANIFEST.in b/logging/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/logging/MANIFEST.in +++ b/logging/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/logging/requirements.txt b/logging/requirements.txt new file mode 100644 index 000000000000..712740392279 --- /dev/null +++ b/logging/requirements.txt @@ -0,0 +1,5 @@ +google-cloud-core >= 0.20.0 +grpcio >= 1.0.0, < 2.0dev +google-gax >= 0.14.1, < 0.15dev +gapic-google-logging-v2 >= 0.10.1, < 0.11dev +grpc-google-logging-v2 >= 0.10.1, < 0.11dev diff --git a/logging/setup.py b/logging/setup.py index 0a0255553e24..64b2ff9304bb 100644 --- a/logging/setup.py +++ b/logging/setup.py @@ -49,13 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', - 'grpcio >= 1.0.0, < 2.0dev', - 'google-gax >= 0.14.1, < 0.15dev', - 'gapic-google-logging-v2 >= 0.10.1, < 0.11dev', - 'grpc-google-logging-v2 >= 0.10.1, < 0.11dev', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-logging', diff --git a/monitoring/MANIFEST.in b/monitoring/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/monitoring/MANIFEST.in +++ b/monitoring/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/monitoring/requirements.txt b/monitoring/requirements.txt new file mode 100644 index 000000000000..db922a2bfe81 --- /dev/null +++ b/monitoring/requirements.txt @@ -0,0 +1 @@ +google-cloud-core >= 0.20.0 diff --git a/monitoring/setup.py b/monitoring/setup.py index 0fcbbc379d02..641c92c70992 100644 --- a/monitoring/setup.py +++ b/monitoring/setup.py @@ -49,9 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-monitoring', diff --git a/pubsub/MANIFEST.in b/pubsub/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/pubsub/MANIFEST.in +++ b/pubsub/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/pubsub/requirements.txt b/pubsub/requirements.txt new file mode 100644 index 000000000000..ea7e263e3bb7 --- /dev/null +++ b/pubsub/requirements.txt @@ -0,0 +1,5 @@ +google-cloud-core >= 0.20.0 +grpcio >= 1.0.0, < 2.0dev +google-gax >= 0.14.1, < 0.15dev +gapic-google-pubsub-v1 >= 0.10.1, < 0.11dev +grpc-google-pubsub-v1 >= 0.10.1, < 0.11dev diff --git a/pubsub/setup.py b/pubsub/setup.py index 6accc0ae8138..85bb25c17018 100644 --- a/pubsub/setup.py +++ b/pubsub/setup.py @@ -49,13 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', - 'grpcio >= 1.0.0, < 2.0dev', - 'google-gax >= 0.14.1, < 0.15dev', - 'gapic-google-pubsub-v1 >= 0.10.1, < 0.11dev', - 'grpc-google-pubsub-v1 >= 0.10.1, < 0.11dev', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-pubsub', diff --git a/resource_manager/MANIFEST.in b/resource_manager/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/resource_manager/MANIFEST.in +++ b/resource_manager/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/resource_manager/requirements.txt b/resource_manager/requirements.txt new file mode 100644 index 000000000000..db922a2bfe81 --- /dev/null +++ b/resource_manager/requirements.txt @@ -0,0 +1 @@ +google-cloud-core >= 0.20.0 diff --git a/resource_manager/setup.py b/resource_manager/setup.py index 063b4dcbd16f..17f5a532b55c 100644 --- a/resource_manager/setup.py +++ b/resource_manager/setup.py @@ -49,9 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-resource-manager', diff --git a/speech/MANIFEST.in b/speech/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/speech/MANIFEST.in +++ b/speech/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/speech/requirements.txt b/speech/requirements.txt new file mode 100644 index 000000000000..db922a2bfe81 --- /dev/null +++ b/speech/requirements.txt @@ -0,0 +1 @@ +google-cloud-core >= 0.20.0 diff --git a/speech/setup.py b/speech/setup.py index c02aeaad3e9d..c09f82039f83 100644 --- a/speech/setup.py +++ b/speech/setup.py @@ -49,9 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-speech', diff --git a/storage/MANIFEST.in b/storage/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/storage/MANIFEST.in +++ b/storage/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/storage/requirements.txt b/storage/requirements.txt new file mode 100644 index 000000000000..db922a2bfe81 --- /dev/null +++ b/storage/requirements.txt @@ -0,0 +1 @@ +google-cloud-core >= 0.20.0 diff --git a/storage/setup.py b/storage/setup.py index 3632105ff6f9..a60f6d638a9d 100644 --- a/storage/setup.py +++ b/storage/setup.py @@ -49,9 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-storage', diff --git a/translate/MANIFEST.in b/translate/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/translate/MANIFEST.in +++ b/translate/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/translate/requirements.txt b/translate/requirements.txt new file mode 100644 index 000000000000..db922a2bfe81 --- /dev/null +++ b/translate/requirements.txt @@ -0,0 +1 @@ +google-cloud-core >= 0.20.0 diff --git a/translate/setup.py b/translate/setup.py index c135b122e920..d2255b646cdb 100644 --- a/translate/setup.py +++ b/translate/setup.py @@ -49,9 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-translate', diff --git a/vision/MANIFEST.in b/vision/MANIFEST.in index cb3a2b9ef4fa..0a04c904975e 100644 --- a/vision/MANIFEST.in +++ b/vision/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst +include requirements.txt graft google graft unit_tests global-exclude *.pyc diff --git a/vision/requirements.txt b/vision/requirements.txt new file mode 100644 index 000000000000..db922a2bfe81 --- /dev/null +++ b/vision/requirements.txt @@ -0,0 +1 @@ +google-cloud-core >= 0.20.0 diff --git a/vision/setup.py b/vision/setup.py index 59e8cec8c67f..9658fe043180 100644 --- a/vision/setup.py +++ b/vision/setup.py @@ -49,9 +49,12 @@ } -REQUIREMENTS = [ - 'google-cloud-core >= 0.20.0', -] +# NOTE: This assumes that no comments or other extra text +# is in the requirements.txt file. +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as file_obj: + REQUIREMENTS = file_obj.read() + REQUIREMENTS = REQUIREMENTS.strip().split('\n') + setup( name='google-cloud-vision', From f2b87af92719a34525a2170ddd4020a023d0667e Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 3 Oct 2016 20:51:54 -0700 Subject: [PATCH 2/2] On Travis, follow package dependency graph to expand package list. --- scripts/generate_json_docs.py | 10 ++-- scripts/make_datastore_grpc.py | 8 +-- scripts/run_pylint.py | 3 +- scripts/run_unit_tests.py | 11 +++-- scripts/script_utils.py | 79 ++++++++++++++++++++++++++++++ scripts/verify_included_modules.py | 8 +-- 6 files changed, 101 insertions(+), 18 deletions(-) diff --git a/scripts/generate_json_docs.py b/scripts/generate_json_docs.py index 6bc303219df4..f8376568d8e5 100644 --- a/scripts/generate_json_docs.py +++ b/scripts/generate_json_docs.py @@ -26,6 +26,7 @@ from parinx.errors import MethodParsingException import six +from script_utils import PROJECT_ROOT from verify_included_modules import get_public_modules @@ -601,7 +602,7 @@ def main(): parser.add_argument('--tag', help='The version of the documentation.', default='master') parser.add_argument('--basepath', help='Path to the library.', - default=os.path.join(os.path.dirname(__file__), '..')) + default=PROJECT_ROOT) parser.add_argument('--show-toc', help='Prints partial table of contents', default=False) args = parser.parse_args() @@ -635,10 +636,9 @@ def main(): } } - BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) - BASE_JSON_DOCS_DIR = os.path.join(BASE_DIR, 'docs', 'json') + BASE_JSON_DOCS_DIR = os.path.join(PROJECT_ROOT, 'docs', 'json') - DOCS_BUILD_DIR = os.path.join(BASE_DIR, 'docs', '_build') + DOCS_BUILD_DIR = os.path.join(PROJECT_ROOT, 'docs', '_build') JSON_DOCS_DIR = os.path.join(DOCS_BUILD_DIR, 'json', args.tag) LIB_DIR = os.path.abspath(args.basepath) @@ -646,7 +646,7 @@ def main(): public_mods = get_public_modules(library_dir, base_package='google.cloud') - generate_module_docs(public_mods, JSON_DOCS_DIR, BASE_DIR, toc) + generate_module_docs(public_mods, JSON_DOCS_DIR, PROJECT_ROOT, toc) generate_doc_types_json(public_mods, os.path.join(JSON_DOCS_DIR, 'types.json')) package_files(JSON_DOCS_DIR, DOCS_BUILD_DIR, BASE_JSON_DOCS_DIR) diff --git a/scripts/make_datastore_grpc.py b/scripts/make_datastore_grpc.py index 33db9b313a0f..b0e67ffc7f62 100644 --- a/scripts/make_datastore_grpc.py +++ b/scripts/make_datastore_grpc.py @@ -20,13 +20,13 @@ import sys import tempfile +from script_utils import PROJECT_ROOT -ROOT_DIR = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..')) -PROTOS_DIR = os.path.join(ROOT_DIR, 'googleapis-pb') + +PROTOS_DIR = os.path.join(PROJECT_ROOT, 'googleapis-pb') PROTO_PATH = os.path.join(PROTOS_DIR, 'google', 'datastore', 'v1', 'datastore.proto') -GRPC_ONLY_FILE = os.path.join(ROOT_DIR, 'datastore', +GRPC_ONLY_FILE = os.path.join(PROJECT_ROOT, 'datastore', 'google', 'cloud', 'datastore', '_generated', 'datastore_grpc_pb2.py') GRPCIO_VIRTUALENV = os.getenv('GRPCIO_VIRTUALENV') diff --git a/scripts/run_pylint.py b/scripts/run_pylint.py index b9b2a7731c23..c18204c50e3c 100644 --- a/scripts/run_pylint.py +++ b/scripts/run_pylint.py @@ -30,6 +30,7 @@ import sys from script_utils import get_affected_files +from script_utils import PROJECT_ROOT IGNORED_DIRECTORIES = [ @@ -44,7 +45,7 @@ os.path.join('google', 'cloud', '__init__.py'), 'setup.py', ] -SCRIPTS_DIR = os.path.abspath(os.path.dirname(__file__)) +SCRIPTS_DIR = os.path.join(PROJECT_ROOT, 'scripts') PRODUCTION_RC = os.path.join(SCRIPTS_DIR, 'pylintrc_default') TEST_RC = os.path.join(SCRIPTS_DIR, 'pylintrc_reduced') TEST_DISABLED_MESSAGES = [ diff --git a/scripts/run_unit_tests.py b/scripts/run_unit_tests.py index 4254f2f8e6b9..44334a6f3f0b 100644 --- a/scripts/run_unit_tests.py +++ b/scripts/run_unit_tests.py @@ -27,14 +27,14 @@ import sys from script_utils import check_output +from script_utils import follow_dependencies from script_utils import get_changed_packages from script_utils import in_travis from script_utils import in_travis_pr +from script_utils import PROJECT_ROOT from script_utils import travis_branch -PROJECT_ROOT = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..')) IGNORED_DIRECTORIES = ( 'appveyor', 'docs', @@ -121,7 +121,9 @@ def get_test_packages(): * Check command line for packages passed in as positional arguments * Check if in Travis, then limit the subset based on changes in a Pull Request ("push" builds to branches may not have - any filtering) + any filtering). Once the filtered list of **changed** packages + is found, the package dependency graph is used to add any + additional packages which depend on the changed packages. * Just use all packages :rtype: list @@ -136,7 +138,8 @@ def get_test_packages(): verify_packages(args.packages, all_packages) return sorted(args.packages) elif in_travis(): - return get_travis_directories(all_packages) + changed_packages = get_travis_directories(all_packages) + return follow_dependencies(changed_packages, all_packages) else: return all_packages diff --git a/scripts/script_utils.py b/scripts/script_utils.py index 370408e6faa3..9c8c66fc1ea8 100644 --- a/scripts/script_utils.py +++ b/scripts/script_utils.py @@ -20,11 +20,14 @@ import subprocess +PROJECT_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) LOCAL_REMOTE_ENV = 'GOOGLE_CLOUD_TESTING_REMOTE' LOCAL_BRANCH_ENV = 'GOOGLE_CLOUD_TESTING_BRANCH' IN_TRAVIS_ENV = 'TRAVIS' TRAVIS_PR_ENV = 'TRAVIS_PULL_REQUEST' TRAVIS_BRANCH_ENV = 'TRAVIS_BRANCH' +PACKAGE_PREFIX = 'google-cloud-' def in_travis(): @@ -213,3 +216,79 @@ def get_affected_files(allow_limited=True): result = subprocess.check_output(['git', 'ls-files']) return result.rstrip('\n').split('\n'), diff_base + + +def get_required_packages(file_contents): + """Get required packages from a requirements.txt file. + + .. note:: + + This could be done in a bit more complete way via + https://pypi.python.org/pypi/requirements-parser + + :type file_contents: str + :param file_contents: The contents of a requirements.txt file. + + :rtype: list + :returns: The list of required packages. + """ + requirements = file_contents.strip().split('\n') + result = [] + for required in requirements: + parts = required.split() + result.append(parts[0]) + return result + + +def get_dependency_graph(package_list): + """Get a directed graph of package dependencies. + + :type package_list: list + :param package_list: The list of **all** valid packages. + + :rtype: dict + :returns: A dictionary where keys are packages and values are + the set of packages that depend on the key. + """ + result = {package: set() for package in package_list} + for package in package_list: + reqs_file = os.path.join(PROJECT_ROOT, package, + 'requirements.txt') + with open(reqs_file, 'r') as file_obj: + file_contents = file_obj.read() + + requirements = get_required_packages(file_contents) + for requirement in requirements: + if not requirement.startswith(PACKAGE_PREFIX): + continue + _, req_package = requirement.split(PACKAGE_PREFIX) + req_package = req_package.replace('-', '_') + result[req_package].add(package) + + return result + + +def follow_dependencies(subset, package_list): + """Get a directed graph of packages dependency. + + :type subset: list + :param subset: List of a subset of package names. + + :type package_list: list + :param package_list: The list of **all** valid packages. + + :rtype: list + :returns: An expanded list of packages containing everything + in ``subset`` and any packages that depend on those. + """ + dependency_graph = get_dependency_graph(package_list) + + curr_pkgs = None + updated_pkgs = set(subset) + while curr_pkgs != updated_pkgs: + curr_pkgs = updated_pkgs + updated_pkgs = set(curr_pkgs) + for package in curr_pkgs: + updated_pkgs.update(dependency_graph[package]) + + return sorted(curr_pkgs) diff --git a/scripts/verify_included_modules.py b/scripts/verify_included_modules.py index f528966278f3..ed447585e2d5 100644 --- a/scripts/verify_included_modules.py +++ b/scripts/verify_included_modules.py @@ -24,10 +24,10 @@ from sphinx.ext.intersphinx import fetch_inventory +from script_utils import PROJECT_ROOT -BASE_DIR = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..')) -DOCS_DIR = os.path.join(BASE_DIR, 'docs') + +DOCS_DIR = os.path.join(PROJECT_ROOT, 'docs') IGNORED_PREFIXES = ('test_', '_') IGNORED_MODULES = frozenset([ 'google.cloud.__init__', @@ -153,7 +153,7 @@ def verify_modules(build_root='_build'): public_mods = set() for package in PACKAGES: - library_dir = os.path.join(BASE_DIR, package, 'google', 'cloud') + library_dir = os.path.join(PROJECT_ROOT, package, 'google', 'cloud') package_mods = get_public_modules(library_dir, base_package='google.cloud') public_mods.update(package_mods)