From 50eacb73305f17efd19913d78ccdc646af8e5243 Mon Sep 17 00:00:00 2001 From: Daymon <17409137+daymxn@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:32:51 -0600 Subject: [PATCH] Add clean command to fireci (#6563) Per [b/381270432](https://b.corp.google.com/issues/381270432), This adds a command to our `fireci` cli tool that can work as an alternative to `gradle clean`. There seems to be some dependency issues preventing `gradle clean` from completely correctly, but with the usage of `fireci`- we can avoid working within gradle entirely. Furthermore, this command comes with flags for deeper cleaning of gradle's caches. I've also updated the dependencies that fireci was using, as I ran into a variety of issues trying to use it "as-is" on an M1 machine. The updated dependencies seem to include fixes for such issues. Command usage: ```sh fireci clean --help ``` --- ci/README.md | 25 ++++++- ci/fireci/fireci/ci_utils.py | 27 +++++++ ci/fireci/fireci/dir_utils.py | 27 +++++++ ci/fireci/fireci/internal.py | 13 +++- ci/fireci/fireci/main.py | 7 +- ci/fireci/fireciplugins/clean.py | 118 +++++++++++++++++++++++++++++++ ci/fireci/setup.cfg | 23 +++--- 7 files changed, 224 insertions(+), 16 deletions(-) create mode 100644 ci/fireci/fireciplugins/clean.py diff --git a/ci/README.md b/ci/README.md index d1546d1ddbb..a4b4eb4799b 100644 --- a/ci/README.md +++ b/ci/README.md @@ -4,7 +4,7 @@ This directory contains tooling used to run Continuous Integration tasks. ## Prerequisites -- Requires python3.5+ and setuptools to be installed. +- Requires python3.9+ and setuptools to be installed. ## Setup @@ -22,3 +22,26 @@ This directory contains tooling used to run Continuous Integration tasks. ``` fireci --help ``` + +## Uninstall + +If you run into any issues and need to re-install, or uninstall the package, you can do so +by uninstalling the `fireci` package. + +```shell +pip3 uninstall fireci -y +``` + +## Debug + +By default, if you're not running `fireci` within the context of CI, the minimum log level is set +to `INFO`. + +To manually set the level to `DEBUG`, you can use the `--debug` flag. + +```shell +fireci --debug clean +``` + +> ![NOTE] +> The `--debug` flag must come _before_ the command. diff --git a/ci/fireci/fireci/ci_utils.py b/ci/fireci/fireci/ci_utils.py index 12ac98b93f6..1b9e0477e75 100644 --- a/ci/fireci/fireci/ci_utils.py +++ b/ci/fireci/fireci/ci_utils.py @@ -16,6 +16,8 @@ import os import subprocess +from typing import List, Tuple, Union + _logger = logging.getLogger('fireci.ci_utils') @@ -61,3 +63,28 @@ def gcloud_identity_token(): """Returns an identity token with the current gcloud service account.""" result = subprocess.run(['gcloud', 'auth', 'print-identity-token'], stdout=subprocess.PIPE, check=True) return result.stdout.decode('utf-8').strip() + +def get_projects(file_path: str = "subprojects.cfg") -> List[str]: + """Parses the specified file for a list of projects in the repo.""" + with open(file_path, 'r') as file: + stripped_lines = [line.strip() for line in file] + return [line for line in stripped_lines if line and not line.startswith('#')] + +def counts(arr: List[Union[bool, int]]) -> Tuple[int, int]: + """Given an array of booleans and ints, returns a tuple mapping of [true, false]. + Positive int values add to the `true` count while values less than one add to `false`. + """ + true_count = 0 + false_count = 0 + for value in arr: + if isinstance(value, bool): + if value: + true_count += 1 + else: + false_count += 1 + elif value >= 1: + true_count += value + else: + false_count += abs(value) if value < 0 else 1 + + return true_count, false_count diff --git a/ci/fireci/fireci/dir_utils.py b/ci/fireci/fireci/dir_utils.py index c5aea659d06..bb0dbb0fb44 100644 --- a/ci/fireci/fireci/dir_utils.py +++ b/ci/fireci/fireci/dir_utils.py @@ -15,6 +15,9 @@ import contextlib import logging import os +import pathlib +import shutil +import glob _logger = logging.getLogger('fireci.dir_utils') @@ -30,3 +33,27 @@ def chdir(directory): finally: _logger.debug(f'Restoring directory to: {original_dir} ...') os.chdir(original_dir) + +def rmdir(path: str) -> bool: + """Recursively deletes a directory, and returns a boolean indicating if the dir was deleted.""" + dir = pathlib.Path(path) + if not dir.exists(): + _logger.debug(f"Directory already deleted: {dir}") + return False + + _logger.debug(f"Deleting directory: {dir}") + shutil.rmtree(dir) + return True + +def rmglob(pattern: str) -> int: + """Deletes all files that match a given pattern, and returns the amount of (root) files deleted""" + files = glob.glob(os.path.expanduser(pattern)) + for file in files: + path = pathlib.Path(file) + if path.is_dir(): + rmdir(file) + else: + _logger.debug(f"Deleting file: {path}") + os.remove(path) + + return len(files) diff --git a/ci/fireci/fireci/internal.py b/ci/fireci/fireci/internal.py index 0950d770fc2..7078528c512 100644 --- a/ci/fireci/fireci/internal.py +++ b/ci/fireci/fireci/internal.py @@ -58,6 +58,11 @@ class _CommonOptions: @click.group() +@click.option( + '--debug/--no-debug', + help='Set the min loglevel to debug.', + default=False +) @click.option( '--artifact-target-dir', default='_artifacts', @@ -83,7 +88,7 @@ def main(options, **kwargs): setattr(options, k, v) -def ci_command(name=None, cls=click.Command, group=main): +def ci_command(name=None, cls=click.Command, group=main, epilog=None): """Decorator to use for CI commands. The differences from the standard @click.command are: @@ -94,15 +99,19 @@ def ci_command(name=None, cls=click.Command, group=main): :param name: Optional name of the task. Defaults to the function name that is decorated with this decorator. :param cls: Specifies whether the func is a command or a command group. Defaults to `click.Command`. :param group: Specifies the group the command belongs to. Defaults to the `main` command group. + :param epilog: Specifies epilog text to show at the end of the help text. """ def ci_command(f): actual_name = f.__name__ if name is None else name - @click.command(name=actual_name, cls=cls, help=f.__doc__) + @click.command(name=actual_name, cls=cls, help=f.__doc__, epilog=epilog) @_pass_options @click.pass_context def new_func(ctx, options, *args, **kwargs): + if options.debug: + logging.getLogger('fireci').setLevel(logging.DEBUG) + with _artifact_handler( options.artifact_target_dir, options.artifact_patterns, diff --git a/ci/fireci/fireci/main.py b/ci/fireci/fireci/main.py index 9348f69b02c..957cca3c3c8 100644 --- a/ci/fireci/fireci/main.py +++ b/ci/fireci/fireci/main.py @@ -20,14 +20,17 @@ from .internal import main # Unnecessary on CI as GitHub Actions provides them already. -asctime_place_holder = '' if os.getenv('CI') else '%(asctime)s ' +is_ci = os.getenv('CI') +asctime_place_holder = '' if is_ci else '%(asctime)s ' log_format = f'[%(levelname).1s] {asctime_place_holder}%(name)s: %(message)s' logging.basicConfig( datefmt='%Y-%m-%d %H:%M:%S %z %Z', format=log_format, level=logging.INFO, ) -logging.getLogger('fireci').setLevel(logging.DEBUG) + +level = logging.DEBUG if is_ci else logging.INFO +logging.getLogger('fireci').setLevel(level) plugins.discover() diff --git a/ci/fireci/fireciplugins/clean.py b/ci/fireci/fireciplugins/clean.py new file mode 100644 index 00000000000..9f2cd6af9a5 --- /dev/null +++ b/ci/fireci/fireciplugins/clean.py @@ -0,0 +1,118 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +import logging + +from fireci import ci_command +from fireci import ci_utils +from fireci import dir_utils +from typing import Tuple, List, Callable, Union +from termcolor import colored + +log = logging.getLogger('fireci.clean') + +@click.argument("projects", + nargs=-1, + type=click.Path(), + required=False +) +@click.option('--gradle/--no-gradle', default=False, help="Delete the local .gradle caches.") +@click.option('--build/--no-build', default=True, help="Delete the local build caches.") +@click.option('--transforms/--no-transforms', default=False, help="Delete the system-wide transforms cache.") +@click.option('--build-cache/--no-build-cache', default=False, help="Delete the system-wide build cache.") + +@click.option('--deep/--no-deep', default=False, help="Delete all of the system-wide files for gradle.") +@click.option('--cache/--no-cache', default=False, help="Delete all of the system-wide caches for gradle.") +@ci_command(epilog=""" + Clean a subset of projects: + + \b + $ fireci clean firebase-common + $ fireci clean firebase-common firebase-vertexai + + Clean all projects: + + $ fireci clean +""") +def clean(projects, gradle, build, transforms, build_cache, deep, cache): + """ + Delete files cached by gradle. + + Alternative to the standard `gradlew clean`, which runs outside the scope of gradle, + and provides deeper cache cleaning capabilities. + """ + if not projects: + log.debug("No projects specified, so we're defaulting to all projects.") + projects = ci_utils.get_projects() + + cache = cache or deep + gradle = gradle or cache + + cleaners = [] + + if build: + cleaners.append(delete_build) + if gradle: + cleaners.append(delete_gradle) + + results = [call_and_sum(projects, cleaner) for cleaner in cleaners] + local_count = tuple(map(sum, zip(*results))) + + cleaners = [] + + if deep: + cleaners.append(delete_deep) + elif cache: + cleaners.append(delete_cache) + else: + if transforms: + cleaners.append(delete_transforms) + if build_cache: + cleaners.append(delete_build_cache) + + results = [cleaner() for cleaner in cleaners] + system_count = ci_utils.counts(results) + + [deleted, skipped] = tuple(a + b for a, b in zip(local_count, system_count)) + + log.info(f""" + Clean results: + + {colored("Deleted:", None, attrs=["bold"])} {colored(deleted, "red")} + {colored("Already deleted:", None, attrs=["bold"])} {colored(skipped, "grey")} + """) + + +def call_and_sum(variables: List[str], func: Callable[[str], Union[bool, int]]) -> Tuple[int, int]: + results = list(map(lambda var: func(var), variables)) + return ci_utils.counts(results) + +def delete_build(dir: str) -> bool: + return dir_utils.rmdir(f"{dir}/build") + +def delete_gradle(dir: str) -> bool: + return dir_utils.rmdir(f"{dir}/.gradle") + +def delete_transforms() -> int: + return dir_utils.rmglob("~/.gradle/caches/transforms-*") + +def delete_build_cache() -> int: + return dir_utils.rmglob("~/.gradle/caches/build-cache-*") + +def delete_deep() -> bool: + return dir_utils.rmdir("~/.gradle") + +def delete_cache() -> bool: + return dir_utils.rmdir("~/.gradle/caches") diff --git a/ci/fireci/setup.cfg b/ci/fireci/setup.cfg index 466898d3cb6..7b49519871c 100644 --- a/ci/fireci/setup.cfg +++ b/ci/fireci/setup.cfg @@ -4,17 +4,18 @@ version = 0.1 [options] install_requires = - protobuf==3.19 - click==8.1.3 - google-cloud-storage==2.5.0 - mypy==0.991 - numpy==1.23.1 - pandas==1.5.1 - PyGithub==1.55 - pystache==0.6.0 - requests==2.23.0 - seaborn==0.12.1 - PyYAML==6.0.0 + protobuf==3.20.3 + click==8.1.7 + google-cloud-storage==2.18.2 + mypy==1.6.0 + numpy==1.24.4 + pandas==1.5.3 + PyGithub==1.58.2 + pystache==0.6.0 + requests==2.31.0 + seaborn==0.12.2 + PyYAML==6.0.1 + termcolor==2.4.0 [options.extras_require] test =