Add clean command to fireci (#6563)

Per [b/381270432](https://b.corp.google.com/issues/381270432),

This adds a command to our `fireci` cli tool that can work as an
alternative to `gradle clean`. There seems to be some dependency issues
preventing `gradle clean` from completely correctly, but with the usage
of `fireci`- we can avoid working within gradle entirely.

Furthermore, this command comes with flags for deeper cleaning of
gradle's caches.

I've also updated the dependencies that fireci was using, as I ran into
a variety of issues trying to use it "as-is" on an M1 machine. The
updated dependencies seem to include fixes for such issues.

Command usage:
```sh
fireci clean --help
```

ci/README.md

@@ -4,7 +4,7 @@ This directory contains tooling used to run Continuous Integration tasks.
 
 ## Prerequisites
 
-- Requires python3.5+ and setuptools to be installed.
+- Requires python3.9+ and setuptools to be installed.
 
 ## Setup
 
@@ -22,3 +22,26 @@ This directory contains tooling used to run Continuous Integration tasks.
   ```
   fireci --help
   ```
+
+## Uninstall
+
+If you run into any issues and need to re-install, or uninstall the package, you can do so
+by uninstalling the `fireci` package.
+
+```shell
+pip3 uninstall fireci -y
+```
+
+## Debug
+
+By default, if you're not running `fireci` within the context of CI, the minimum log level is set
+to `INFO`.
+
+To manually set the level to `DEBUG`, you can use the `--debug` flag.
+
+```shell
+fireci --debug clean
+```
+
+> ![NOTE]
+> The `--debug` flag must come _before_ the command.

ci/fireci/fireci/ci_utils.py

@@ -16,6 +16,8 @@
 import os
 import subprocess
 
+from typing import List, Tuple, Union
+
 _logger = logging.getLogger('fireci.ci_utils')
 
 
@@ -61,3 +63,28 @@ def gcloud_identity_token():
   """Returns an identity token with the current gcloud service account."""
   result = subprocess.run(['gcloud', 'auth', 'print-identity-token'], stdout=subprocess.PIPE, check=True)
   return result.stdout.decode('utf-8').strip()
+
+def get_projects(file_path: str = "subprojects.cfg") -> List[str]:
+  """Parses the specified file for a list of projects in the repo."""
+  with open(file_path, 'r') as file:
+    stripped_lines = [line.strip() for line in file]
+    return [line for line in stripped_lines if line and not line.startswith('#')]
+
+def counts(arr: List[Union[bool, int]]) -> Tuple[int, int]:
+  """Given an array of booleans and ints, returns a tuple mapping of [true, false]. 
+  Positive int values add to the `true` count while values less than one add to `false`.
+  """
+  true_count = 0
+  false_count = 0
+  for value in arr:
+    if isinstance(value, bool):
+      if value:
+        true_count += 1
+      else:
+        false_count += 1
+    elif value >= 1:
+      true_count += value
+    else:
+      false_count += abs(value) if value < 0 else 1
+
+  return true_count, false_count

ci/fireci/fireci/dir_utils.py

@@ -15,6 +15,9 @@
 import contextlib
 import logging
 import os
+import pathlib
+import shutil
+import glob
 
 _logger = logging.getLogger('fireci.dir_utils')
 
@@ -30,3 +33,27 @@ def chdir(directory):
   finally:
     _logger.debug(f'Restoring directory to: {original_dir} ...')
     os.chdir(original_dir)
+
+def rmdir(path: str) -> bool:
+  """Recursively deletes a directory, and returns a boolean indicating if the dir was deleted."""
+  dir = pathlib.Path(path)
+  if not dir.exists():
+    _logger.debug(f"Directory already deleted: {dir}")
+    return False
+
+  _logger.debug(f"Deleting directory: {dir}")
+  shutil.rmtree(dir)
+  return True
+
+def rmglob(pattern: str) -> int:
+  """Deletes all files that match a given pattern, and returns the amount of (root) files deleted"""
+  files = glob.glob(os.path.expanduser(pattern))
+  for file in files:
+    path = pathlib.Path(file)
+    if path.is_dir():
+      rmdir(file)
+    else:
+      _logger.debug(f"Deleting file: {path}")
+      os.remove(path)
+
+  return len(files)

ci/fireci/fireci/internal.py

@@ -58,6 +58,11 @@ class _CommonOptions:
 
 
 @click.group()
+@click.option(
+  '--debug/--no-debug',
+  help='Set the min loglevel to debug.',
+  default=False
+)
 @click.option(
     '--artifact-target-dir',
     default='_artifacts',
@@ -83,7 +88,7 @@ def main(options, **kwargs):
     setattr(options, k, v)
 
 
-def ci_command(name=None, cls=click.Command, group=main):
+def ci_command(name=None, cls=click.Command, group=main, epilog=None):
   """Decorator to use for CI commands.
 
        The differences from the standard @click.command are:
@@ -94,15 +99,19 @@ def ci_command(name=None, cls=click.Command, group=main):
        :param name:  Optional name of the task. Defaults to the function name that is decorated with this decorator.
        :param cls:   Specifies whether the func is a command or a command group. Defaults to `click.Command`.
        :param group: Specifies the group the command belongs to. Defaults to the `main` command group.
+       :param epilog: Specifies epilog text to show at the end of the help text.
     """
 
   def ci_command(f):
     actual_name = f.__name__ if name is None else name
 
-    @click.command(name=actual_name, cls=cls, help=f.__doc__)
+    @click.command(name=actual_name, cls=cls, help=f.__doc__, epilog=epilog)
     @_pass_options
     @click.pass_context
     def new_func(ctx, options, *args, **kwargs):
+      if options.debug:
+        logging.getLogger('fireci').setLevel(logging.DEBUG)
+
       with _artifact_handler(
           options.artifact_target_dir,
           options.artifact_patterns,

ci/fireci/fireci/main.py

@@ -20,14 +20,17 @@
 from .internal import main
 
 # Unnecessary on CI as GitHub Actions provides them already.
-asctime_place_holder = '' if os.getenv('CI') else '%(asctime)s '
+is_ci = os.getenv('CI')
+asctime_place_holder = '' if is_ci else '%(asctime)s '
 log_format = f'[%(levelname).1s] {asctime_place_holder}%(name)s: %(message)s'
 logging.basicConfig(
     datefmt='%Y-%m-%d %H:%M:%S %z %Z',
     format=log_format,
     level=logging.INFO,
 )
-logging.getLogger('fireci').setLevel(logging.DEBUG)
+
+level = logging.DEBUG if is_ci else logging.INFO
+logging.getLogger('fireci').setLevel(level)
 
 plugins.discover()
 

ci/fireci/fireciplugins/clean.py

@@ -0,0 +1,118 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import logging
+
+from fireci import ci_command
+from fireci import ci_utils
+from fireci import dir_utils
+from typing import Tuple, List, Callable, Union
+from termcolor import colored
+
+log = logging.getLogger('fireci.clean')
+
+@click.argument("projects", 
+  nargs=-1,
+  type=click.Path(), 
+  required=False
+)
+@click.option('--gradle/--no-gradle', default=False, help="Delete the local .gradle caches.")
+@click.option('--build/--no-build', default=True, help="Delete the local build caches.")
+@click.option('--transforms/--no-transforms', default=False, help="Delete the system-wide transforms cache.")
+@click.option('--build-cache/--no-build-cache', default=False, help="Delete the system-wide build cache.")
+
+@click.option('--deep/--no-deep', default=False, help="Delete all of the system-wide files for gradle.")
+@click.option('--cache/--no-cache', default=False, help="Delete all of the system-wide caches for gradle.")
+@ci_command(epilog="""
+  Clean a subset of projects:
+
+  \b
+  $ fireci clean firebase-common
+  $ fireci clean firebase-common firebase-vertexai
+
+  Clean all projects:
+
+  $ fireci clean
+""")
+def clean(projects, gradle, build, transforms, build_cache, deep, cache):
+  """
+  Delete files cached by gradle.
+
+  Alternative to the standard `gradlew clean`, which runs outside the scope of gradle,
+  and provides deeper cache cleaning capabilities.
+  """
+  if not projects:
+    log.debug("No projects specified, so we're defaulting to all projects.")
+    projects = ci_utils.get_projects()
+
+  cache = cache or deep
+  gradle = gradle or cache
+
+  cleaners = []
+
+  if build:
+    cleaners.append(delete_build)
+  if gradle:
+    cleaners.append(delete_gradle)
+
+  results = [call_and_sum(projects, cleaner) for cleaner in cleaners]
+  local_count = tuple(map(sum, zip(*results)))
+
+  cleaners = []
+
+  if deep:
+    cleaners.append(delete_deep)
+  elif cache:
+    cleaners.append(delete_cache)
+  else:
+    if transforms:
+      cleaners.append(delete_transforms)
+    if build_cache:
+      cleaners.append(delete_build_cache)
+
+  results = [cleaner() for cleaner in cleaners]
+  system_count = ci_utils.counts(results)
+
+  [deleted, skipped] = tuple(a + b for a, b in zip(local_count, system_count))
+
+  log.info(f"""
+  Clean results:
+
+    {colored("Deleted:", None, attrs=["bold"])} {colored(deleted, "red")}
+    {colored("Already deleted:", None, attrs=["bold"])} {colored(skipped, "grey")}
+  """)
+
+
+def call_and_sum(variables: List[str], func: Callable[[str], Union[bool, int]]) -> Tuple[int, int]:
+  results = list(map(lambda var: func(var), variables))
+  return ci_utils.counts(results)
+
+def delete_build(dir: str) -> bool:
+  return dir_utils.rmdir(f"{dir}/build")
+
+def delete_gradle(dir: str) -> bool:
+  return dir_utils.rmdir(f"{dir}/.gradle")
+
+def delete_transforms() -> int:
+  return dir_utils.rmglob("~/.gradle/caches/transforms-*")
+
+def delete_build_cache() -> int:
+  return dir_utils.rmglob("~/.gradle/caches/build-cache-*")
+
+def delete_deep() -> bool:
+  return dir_utils.rmdir("~/.gradle")
+
+def delete_cache() -> bool:
+  return dir_utils.rmdir("~/.gradle/caches")

ci/fireci/setup.cfg

@@ -4,17 +4,18 @@ version = 0.1
 
 [options]
 install_requires =
-        protobuf==3.19
-        click==8.1.3
-        google-cloud-storage==2.5.0
-        mypy==0.991
-        numpy==1.23.1
-        pandas==1.5.1
-        PyGithub==1.55
-        pystache==0.6.0
-        requests==2.23.0
-        seaborn==0.12.1
-        PyYAML==6.0.0
+    protobuf==3.20.3
+    click==8.1.7
+    google-cloud-storage==2.18.2
+    mypy==1.6.0
+    numpy==1.24.4
+    pandas==1.5.3
+    PyGithub==1.58.2
+    pystache==0.6.0
+    requests==2.31.0
+    seaborn==0.12.2
+    PyYAML==6.0.1
+    termcolor==2.4.0
 
 [options.extras_require]
 test =