From c4049a046a798b9035b62f20ab98a07efc33d982 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Fri, 6 Sep 2024 12:22:40 -0500 Subject: [PATCH 1/3] Download counting for SourceForge --- netkan/netkan/download_counter.py | 41 +++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/netkan/netkan/download_counter.py b/netkan/netkan/download_counter.py index e6c8542..a85e36a 100644 --- a/netkan/netkan/download_counter.py +++ b/netkan/netkan/download_counter.py @@ -6,6 +6,7 @@ import urllib.parse from typing import Dict, Tuple, Any, Optional import heapq +from datetime import date import requests @@ -205,10 +206,36 @@ def get_result(self, counts: Optional[Dict[str, int]] = None) -> Dict[str, int]: return counts +class SourceForgeQuerier: + + # https://sourceforge.net/p/forge/documentation/Download%20Stats%20API/ + API_TEMPLATE = Template('https://sourceforge.net/projects/${proj_id}/files/stats/json' + '?start_date=2010-01-01&end_date=${today}' + '&os_by_country=false&period=monthly') + + @classmethod + def get_count(cls, proj_id): + return requests.get(cls.get_query(proj_id), timeout=60).json()['total'] + + @classmethod + def get_query(cls, proj_id): + return cls.API_TEMPLATE.safe_substitute(proj_id=proj_id, + today=date.today().isoformat()) + + @classmethod + def get_result(cls, ident: str, proj_id: str, + counts: Optional[Dict[str, int]] = None) -> Dict[str, int]: + if counts is None: + counts = {} + counts[ident] = counts.get(ident, 0) + cls.get_count(proj_id) + return counts + + class DownloadCounter: GITHUB_PATH_PATTERN = re.compile(r'^/([^/]+)/([^/]+)') SPACEDOCK_PATH_PATTERN = re.compile(r'^/mod/([^/]+)') + SOURCEFORGE_PATH_PATTERN = re.compile(r'^/project/([^/]+)') def __init__(self, game_id: str, ckm_repo: CkanMetaRepo, github_token: str) -> None: self.game_id = game_id @@ -253,6 +280,14 @@ def get_counts(self) -> None: if ia_query.full(): ia_query.get_result(self.counts) ia_query = InternetArchiveBatchedQuery() + elif url_parse.netloc.endswith('.sourceforge.net'): + match = self.SOURCEFORGE_PATH_PATTERN.match(url_parse.path) + if match: + SourceForgeQuerier.get_result(ckan.identifier, match.group(1), + self.counts) + else: + logging.error('Failed to parse SF URL for %s: %s', + ckan.identifier, download) except Exception as exc: # pylint: disable=broad-except # Don't let one bad apple spoil the bunch # Print file path because netkan_dl might be None @@ -296,8 +331,10 @@ def log_top(self, how_many: int) -> None: # This isn't an error, but only errors go to Discord logging.error('Top %s downloads for %s all-time:\n%s\n\n' 'Top %s downloads for %s today:\n%s', - how_many, self.game_id, self._download_summary_table(self.counts, how_many), - how_many, self.game_id, self._download_summary_table(deltas, how_many)) + how_many, self.game_id, + self._download_summary_table(self.counts, how_many), + how_many, self.game_id, + self._download_summary_table(deltas, how_many)) def update_counts(self) -> None: if self.output_file: From 14715d4604e3744cfc48667ebfe5bf9dd46b4623 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Fri, 6 Sep 2024 12:26:06 -0500 Subject: [PATCH 2/3] Types --- netkan/netkan/download_counter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netkan/netkan/download_counter.py b/netkan/netkan/download_counter.py index a85e36a..6fa7098 100644 --- a/netkan/netkan/download_counter.py +++ b/netkan/netkan/download_counter.py @@ -214,11 +214,11 @@ class SourceForgeQuerier: '&os_by_country=false&period=monthly') @classmethod - def get_count(cls, proj_id): + def get_count(cls, proj_id: str) -> int: return requests.get(cls.get_query(proj_id), timeout=60).json()['total'] @classmethod - def get_query(cls, proj_id): + def get_query(cls, proj_id: str) -> str: return cls.API_TEMPLATE.safe_substitute(proj_id=proj_id, today=date.today().isoformat()) From 08ca0e4d19397d78789d20c45aed1a5f50a4274d Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Fri, 6 Sep 2024 12:33:14 -0500 Subject: [PATCH 3/3] Bump action versions --- .github/workflows/build.yml | 6 +++--- .github/workflows/deploy.yml | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de7f9e1..08fbe2d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,16 +8,16 @@ on: - reopened jobs: - main: + Build: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Build Image id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./netkan file: ./netkan/Dockerfile diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8365583..c872cd7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,21 +13,21 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Run tests before deploying - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./netkan file: ./netkan/Dockerfile push: false target: test - name: Build and push NetKAN-infra - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./netkan file: ./netkan/Dockerfile @@ -41,15 +41,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push webhooks-proxy id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./nginx file: ./nginx/Dockerfile