Merge branch 'main' into pre-commit-config-check

.github/workflows/reports_cron.yml

@@ -29,6 +29,9 @@ jobs:
     env:
       ADABOT_GITHUB_USER: ${{ secrets.ADABOT_GITHUB_USER }}
       ADABOT_GITHUB_ACCESS_TOKEN: ${{ secrets.ADABOT_GITHUB_ACCESS_TOKEN }}
+      RTD_TOKEN: ${{ secrets.RTD_TOKEN }}
+      BIGQUERY_PRIVATE_KEY: ${{ secrets.BIGQUERY_PRIVATE_KEY }}
+      BIGQUERY_CLIENT_EMAIL: ${{ secrets.BIGQUERY_CLIENT_EMAIL }}
     steps:
       - name: Dump GitHub context
         env:

.github/workflows/test.yml

@@ -53,6 +53,7 @@ jobs:
         ADABOT_EMAIL: ${{ secrets.ADABOT_EMAIL }}
         ADABOT_GITHUB_USER: ${{ secrets.ADABOT_GITHUB_USER }}
         ADABOT_GITHUB_ACCESS_TOKEN: ${{ secrets.ADABOT_GITHUB_ACCESS_TOKEN }}
+        RTD_TOKEN: ${{ secrets.RTD_TOKEN }}
         REDIS_PORT: ${{ job.services.redis.ports[6379] }}
       run: |
         python3 -u -m pytest

.gitignore

@@ -7,6 +7,7 @@ _build
 .bundles/*
 *.pyc
 .env
+.venv
 env.sh
 *.swp
 .libraries/*

adabot/arduino_libraries.py

@@ -11,7 +11,7 @@
 
 import requests
 
-from adabot import github_requests as github
+from adabot import github_requests as gh_reqs
 
 logger = logging.getLogger(__name__)
 ch = logging.StreamHandler(stream=sys.stdout)
@@ -50,7 +50,7 @@ def list_repos():
     repository state.
     """
     repos = []
-    result = github.get(
+    result = gh_reqs.get(
         "/search/repositories",
         params={
             "q": (
@@ -70,7 +70,7 @@ def list_repos():
         )  # uncomment and comment below, to include all forks
 
         if result.links.get("next"):
-            result = github.get(result.links["next"]["url"])
+            result = gh_reqs.get(result.links["next"]["url"])
         else:
             break
 
@@ -124,7 +124,7 @@ def validate_library_properties(repo):
             lib_version = line[len("version=") :]
             break
 
-    get_latest_release = github.get(
+    get_latest_release = gh_reqs.get(
         "/repos/adafruit/" + repo["name"] + "/releases/latest"
     )
     if get_latest_release.ok:
@@ -151,7 +151,7 @@ def validate_release_state(repo):
     if not is_arduino_library(repo):
         return None
 
-    compare_tags = github.get(
+    compare_tags = gh_reqs.get(
         "/repos/" + repo["full_name"] + "/compare/master..." + repo["tag_name"]
     )
     if not compare_tags.ok:
@@ -187,7 +187,7 @@ def validate_actions(repo):
 
 def validate_example(repo):
     """Validate if a repo has any files in examples directory"""
-    repo_has_ino = github.get("/repos/adafruit/" + repo["name"] + "/contents/examples")
+    repo_has_ino = gh_reqs.get("/repos/adafruit/" + repo["name"] + "/contents/examples")
     return repo_has_ino.ok and len(repo_has_ino.json())
 
 

adabot/circuitpython_bundle.py

@@ -18,7 +18,7 @@
 import sh
 from sh.contrib import git
 
-from adabot import github_requests as github
+from adabot import github_requests as gh_reqs
 from adabot.lib import common_funcs
 
 REDIS = None
@@ -284,7 +284,7 @@ def get_contributors(repo, commit_range):
         author = REDIS.get("github_username:" + author_email)
         committer = REDIS.get("github_username:" + committer_email)
         if not author or not committer:
-            github_commit_info = github.get("/repos/" + repo + "/commits/" + sha)
+            github_commit_info = gh_reqs.get("/repos/" + repo + "/commits/" + sha)
             github_commit_info = github_commit_info.json()
             if github_commit_info["author"]:
                 author = github_commit_info["author"]["login"]
@@ -332,7 +332,7 @@ def new_release(bundle, bundle_path):
     working_directory = os.path.abspath(os.getcwd())
     os.chdir(bundle_path)
     print(bundle)
-    current_release = github.get("/repos/adafruit/{}/releases/latest".format(bundle))
+    current_release = gh_reqs.get("/repos/adafruit/{}/releases/latest".format(bundle))
     last_tag = current_release.json()["tag_name"]
     contributors = get_contributors("adafruit/" + bundle, last_tag + "..")
     added_submodules = []
@@ -430,7 +430,7 @@ def new_release(bundle, bundle_path):
 
     print("Releasing {}".format(release["tag_name"]))
     print(release["body"])
-    response = github.post("/repos/adafruit/" + bundle + "/releases", json=release)
+    response = gh_reqs.post("/repos/adafruit/" + bundle + "/releases", json=release)
     if not response.ok:
         print("Failed to create release")
         print(release)

adabot/circuitpython_libraries.py

@@ -9,11 +9,12 @@
 import datetime
 import inspect
 import logging
+import os
 import re
 import sys
 import traceback
 
-from adabot import github_requests as github
+from adabot import github_requests as gh_reqs
 from adabot import pypi_requests as pypi
 from adabot.lib import circuitpython_library_validators as cirpy_lib_vals
 from adabot.lib import common_funcs
@@ -95,7 +96,7 @@ def run_library_checks(validators, kw_args, error_depth):
     pylint_info = pypi.get("/pypi/pylint/json")
     if pylint_info and pylint_info.ok:
         latest_pylint = pylint_info.json()["info"]["version"]
-    logger.info("Latest pylint is: %s", latest_pylint)
+    # logger.info("Latest pylint is: %s", latest_pylint)
 
     repos = common_funcs.list_repos(
         include_repos=tuple(blinka_repos)
@@ -176,6 +177,13 @@ def run_library_checks(validators, kw_args, error_depth):
     logger.info("")
     logger.info("State of CircuitPython + Libraries + Blinka")
 
+    logger.info("")
+    logger.info("**This report contains information from the previous seven days.**")
+    logger.info(
+        "**Any changes (PRs merged, etc.) made today are not included in this report.**"
+    )
+    logger.info("")
+
     logger.info("### Overall")
     print_pr_overview(lib_insights, core_insights, blinka_insights)
     print_issue_overview(lib_insights, core_insights, blinka_insights)
@@ -195,15 +203,17 @@ def run_library_checks(validators, kw_args, error_depth):
     logger.info("* %s open issues", len(core_insights["open_issues"]))
     logger.info("  * https://github.com/adafruit/circuitpython/issues")
     logger.info("* %s active milestones", len(core_insights["milestones"]))
-    ms_count = 0
-    for milestone in sorted(core_insights["milestones"].keys()):
-        ms_count += core_insights["milestones"][milestone]
-        logger.info(
-            "  * %s: %s open issues", milestone, core_insights["milestones"][milestone]
-        )
+    for milestone, milestone_issue_count in sorted(core_insights["milestones"].items()):
+        logger.info(" * %s: %s open issues", milestone, milestone_issue_count)
+    no_milestone_items = gh_reqs.get(
+        "/repos/adafruit/circuitpython/issues?milestone=none"
+    ).json()
+    no_milestone_issues = [
+        item for item in no_milestone_items if "pull_request" not in item
+    ]
     logger.info(
         "  * %s issues not assigned a milestone",
-        len(core_insights["open_issues"]) - ms_count,
+        len(no_milestone_issues),
     )
     logger.info("")
 
@@ -214,6 +224,26 @@ def run_library_checks(validators, kw_args, error_depth):
     logger.info("* Core download stats available at https://circuitpython.org/stats")
 
     logger.info("")
+
+    # Get PyPI stats
+    if os.environ.get("BIGQUERY_PRIVATE_KEY") and os.environ["BIGQUERY_CLIENT_EMAIL"]:
+        pypi_stats = dl_stats.retrieve_pypi_stats(repos)
+        library_pypi_names = [
+            repo["name"].replace("_", "-").lower() for repo in common_funcs.list_repos()
+        ]
+        library_pypi_stats = []
+        blinka_pypi_stats = None
+        total_library_pypi_stats = 0
+        for stat in pypi_stats:
+            if stat.name == "adafruit-blinka":
+                blinka_pypi_stats = stat
+            if stat.name in library_pypi_names:
+                library_pypi_stats.append(stat)
+            total_library_pypi_stats += stat.num_downloads
+        top_library_pypi_stats = library_pypi_stats[:10]
+    else:
+        pypi_stats = None
+
     logger.info("### Libraries")
     print_pr_overview(lib_insights)
     logger.info("  * Merged pull requests:")
@@ -245,15 +275,40 @@ def run_library_checks(validators, kw_args, error_depth):
 
     logger.info("* https://circuitpython.org/contributing")
 
-    logger.info("Library updates in the last seven days:")
+    logger.info("")
+    logger.info("#### Library PyPI Weekly Download Stats")
+    if pypi_stats:
+        logger.info("* **Total Library Stats**")
+        logger.info(
+            "  * %d PyPI downloads over %d libraries",
+            total_library_pypi_stats,
+            len(library_pypi_names),
+        )
+        logger.info("* **Top 10 Libraries by PyPI Downloads**")
+        for lib_stat in top_library_pypi_stats:
+            logger.info(
+                "  * %s: %d",
+                lib_stat.name,
+                lib_stat.num_downloads,
+            )
+    else:
+        logger.info(
+            "*This info is not printed because the cron is unable to access secrets.*"
+        )
+        logger.info(
+            "*This is expected if this was run as part of the CI for a pull request.*"
+        )
+
+    logger.info("")
+    logger.info("#### Library updates in the last seven days:")
     if len(new_libs) != 0:
-        logger.info("**New Libraries**")
+        logger.info("* **New Libraries**")
         for title, link in new_libs.items():
-            logger.info(" * [%s](%s)", title, link)
+            logger.info("  * [%s](%s)", title, link)
     if len(updated_libs) != 0:
-        logger.info("**Updated Libraries**")
+        logger.info("* **Updated Libraries**")
         for title, link in updated_libs.items():
-            logger.info(" * [%s](%s)", title, link)
+            logger.info("  * [%s](%s)", title, link)
 
     if len(validators) != 0:
         lib_repos = []
@@ -292,7 +347,20 @@ def run_library_checks(validators, kw_args, error_depth):
     print_issue_overview(blinka_insights)
     logger.info("* %s open issues", len(blinka_insights["open_issues"]))
     logger.info("  * https://github.com/adafruit/Adafruit_Blinka/issues")
-    blinka_dl = dl_stats.piwheels_stats().get("adafruit-blinka", {}).get("month", "N/A")
+    blinka_dl = (
+        dl_stats.retrieve_piwheels_stats()
+        .get("adafruit-blinka", {})
+        .get("month", "N/A")
+    )
+    if pypi_stats:
+        logger.info(
+            "* PyPI downloads in the last week: %d",
+            blinka_pypi_stats.num_downloads,
+        )
+    else:
+        logger.info(
+            "* PyPI download stats unavailable - this is normal for CI runs triggered by PRs"
+        )
     logger.info("* Piwheels Downloads in the last month: %s", blinka_dl)
     logger.info("Number of supported boards: %s", blinka_funcs.board_count())
 
@@ -307,7 +375,7 @@ def print_circuitpython_dl_stats():
     #       enable this.
 
     try:
-        response = github.get("/repos/adafruit/circuitpython/releases")
+        response = gh_reqs.get("/repos/adafruit/circuitpython/releases")
     except (ValueError, RuntimeError):
         logger.info("Core CircuitPython GitHub download statistics request failed.")
         return