Automating Quantifying the Commons (Google Summer of Code 2024)

.github/workflows/fetch.yml.disabled

@@ -0,0 +1,46 @@
+name: Fetch Data
+
+on:
+  schedule:
+    # 1:15am onwards, days 1-20, first month of each quarter
+    - cron: '15 1,5,9,13,17,21,23 1-20 1,4,7,10 *'
+  workflow_dispatch:
+
+jobs:
+  fetch:
+    runs-on: ubuntu-latest
+
+    env:
+      GCS_DEVELOPER_KEY: ${{ secrets.GCS_DEVELOPER_KEY }}
+      GCS_CX: ${{ secrets.GCS_CX }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install pipenv
+          pipenv sync --system
+
+      - name: Set PYTHONPATH
+        run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
+
+      - name: Run fetch script
+        run:
+          # python scripts/1-fetch/deviantart_fetched.py
+          # python scripts/1-fetch/flickr_fetched.py
+          # python scripts/1-fetch/github_fetched.py
+          # python scripts/1-fetch/internetarchive_fetched.py
+          # python scripts/1-fetch/metmuseum_fetched.py
+          # python scripts/1-fetch/vimeo_fetched.py
+          # python scripts/1-fetch/wikicommons_fetched.py
+          # python scripts/1-fetch/wikipedia_fetched.py
+          # python scripts/1-fetch/youtube_fetched.py
+          python scripts/1-fetch/gcs_fetched.py

.github/workflows/process.yml.disabled

@@ -0,0 +1,42 @@
+name: Process Data
+
+on:
+  schedule:
+    # 1:15am onwards, days 1-20, second month of each quarter
+    - cron: '15 1,5,9,13,17,21,23 1-20 2,5,8,11 *'
+  workflow_dispatch:
+
+jobs:
+  process:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install pipenv
+          pipenv sync --system
+
+      - name: Set PYTHONPATH
+        run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
+
+      - name: Run process script
+        run:
+          # python scripts/2-process/deviantart_processed.py
+          # python scripts/2-process/flickr_processed.py
+          # python scripts/2-process/github_processed.py
+          # python scripts/2-process/internetarchive_processed.py
+          # python scripts/2-process/metmuseum_processed.py
+          # python scripts/2-process/vimeo_processed.py
+          # python scripts/2-process/wikicommons_processed.py
+          # python scripts/2-process/wikipedia_processed.py
+          # python scripts/2-process/youtube_processed.py
+          python scripts/2-process/gcs_processed.py

.github/workflows/report.yml.disabled

@@ -0,0 +1,43 @@
+name: Generate Report
+
+on:
+  schedule:
+    # 1:15am onwards, days 1-20, third month of each quarter
+    - cron: '15 1,5,9,13,17,21,23 1-20 3,6,9,12 *'
+  workflow_dispatch:
+
+
+jobs:
+  generate-report:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install pipenv
+          pipenv sync --system
+
+      - name: Set PYTHONPATH
+        run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
+
+      - name: Run report script
+        run:
+          # python scripts/3-report/deviantart_reports.py
+          # python scripts/3-report/flickr_reports.py
+          # python scripts/3-report/github_reports.py
+          # python scripts/3-report/internetarchive_reports.py
+          # python scripts/3-report/metmuseum_reports.py
+          # python scripts/3-report/vimeo_reports.py
+          # python scripts/3-report/wikicommons_reports.py
+          # python scripts/3-report/wikipedia_reports.py
+          # python scripts/3-report/youtube_reports.py
+          python scripts/3-report/gcs_reports.py

Pipfile

@@ -5,6 +5,8 @@ name = "pypi"
 
 [packages]
 flickrapi = "*"
+GitPython = "*"
+google-api-python-client = "*"
 internetarchive = "*"
 jupyterlab = ">=3.6.7"
 matplotlib = "*"
@@ -27,3 +29,9 @@ pre-commit = "*"
 
 [requires]
 python_version = "3.11"
+
+[scripts]
+gcs_fetched = "./scripts/1-fetch/gcs_fetched.py"
+flickr_fetched = "./scripts/1-fetch/flickr_fetched.py"
+gcs_processed = "./scripts/2-process/gcs_processed.py"
+gcs_reports = "./scripts/3-report/gcs_reports.py"

README.md

@@ -39,6 +39,49 @@ See [`CONTRIBUTING.md`][org-contrib].
 [org-contrib]: https://github.com/creativecommons/.github/blob/main/CONTRIBUTING.md
 
 
+### Project structure
+
+Please note that in the directory tree below, all instances of `fetch`,
+`process`, and `report` are referring to the three phases of data gathering,
+processing, and report generation.
+
+```
+Quantifying/
+├── .github/
+│   ├── workflows/
+│   │   ├── fetch.yml
+│   │   ├── process.yml
+│   │   ├── report.yml
+│   │   └── static_analysis.yml
+├── data/  # Data generated by script runs
+│   ├── 20XXQX/
+│   │   ├── 1-fetch/
+│   │   ├── 2-process/
+│   │   ├── 3-report/
+│   │   │   └── README.md  # All generated reports are displayed in the README
+│   └── ...
+├── dev/
+├── pre-automation/  # All Quantifying work prior to adding automation system
+├── scripts/  # Run scripts for all phases
+│   ├── 1-fetch/
+│   ├── 2-process/
+│   ├── 3-report/
+│   └── shared.py
+├── .cc-metadata.yml
+├── .flake8  # Python tool configuration
+├── .gitignore
+├── .pre-commit-config.yaml  # Static analysis configuration
+├── LICENSE
+├── Pipfile  # Specifies the project's dependencies and Python version
+├── Pipfile.lock
+├── README.md
+├── env.example
+├── history.md
+├── pyproject.toml  # Python tools configuration
+└── sources.md
+```
+
+
 ## Development
 
 

data/2024Q2/1-fetch/gcs_fetched.csv

@@ -0,0 +1,22 @@
+LICENSE TYPE, No Priori, United States, Canada, India, United Kingdom, Australia, Japan, English, Spanish, French, Arabic, Chinese (Simplified), Indonesian
+https://creativecommons.org/licenses/nc/2.0,57000000,43700000,39400,11500,157000,31100,29300,54600000,1160000,2720000,2960,68100000,101000
+https://creativecommons.org/licenses/sa/1.0,108000000,91000000,34400,19100,158000,43100,25100,104000000,268000,332000,28500,106000000,223000
+https://creativecommons.org/licenses/nd/1.0,222000000,259000000,76700,63400,519000,165000,79600,215000000,492000,383000,69500,222000000,2480000
+https://creativecommons.org/licenses/by-sa/3.0,108000000,88800000,64700,47600,254000,151000,145000,93500000,5010000,3980000,90500,114000000,112000
+https://creativecommons.org/licenses/sampling/1.0,170000000,151000000,26500,23600,263000,56800,34300,170000000,87500,21500,1630,171000000,209000
+https://creativecommons.org/licenses/nc-sa/1.0,24700000,17600000,12600,3800,75300,11600,13400,22600000,67700,272000,1200,24700000,42400
+https://creativecommons.org/licenses/nd-nc/2.0,56900000,43700000,43500,11100,156000,36500,28300,54200000,237000,549000,2660,67600000,11500
+https://creativecommons.org/licenses/by-nd-nc/1.0,16000000,10400000,13500,2900,64700,9990,21300,15500000,63400,28800,1920,16000000,27600
+https://creativecommons.org/licenses/sa/1.0,108000000,91500000,23000,17200,158000,38200,21700,104000000,270000,337000,28600,108000000,224000
+https://creativecommons.org/licenses/nd-nc/1.0,57200000,45500000,23100,8400,123000,21400,30000,56100000,135000,249000,3340,57200000,10200
+https://creativecommons.org/publicdomain/zero/1.0,32700000,30000000,20200,18200,48600,12100,32200,30900000,131000,93400,16100,32700000,10500
+https://creativecommons.org/licenses/nc-sa/2.0,25000000,17700000,19700,4640,80700,14700,11900,23400000,368000,1340000,1490,25000000,36100
+https://creativecommons.org/licenses/sa/1.0,108000000,91400000,23000,17100,159000,38200,21600,104000000,270000,337000,28600,108000000,224000
+https://creativecommons.org/licenses/by-sa/1.0,102000000,85900000,21900,15800,150000,36100,21200,97400000,267000,330000,28300,102000000,219000
+https://creativecommons.org/licenses/by-nd/2.5,49500000,35500000,45100,8640,133000,24600,23600,47800000,367000,37900,2200,49500000,11200
+https://creativecommons.org/licenses/by-nd-nc/1.0,15800000,10300000,15600,2940,66800,10200,21400,15300000,62400,29000,1980,15800000,27500
+https://creativecommons.org/licenses/nd/1.0,220000000,194000000,89900,56500,424000,147000,88400,213000000,490000,297000,69500,220000000,2570000
+https://creativecommons.org/licenses/by-nc-sa/4.0,62900000,44700000,49900,32900,163000,32900,79400,58900000,1020000,3610000,11500,62900000,102000
+https://creativecommons.org/licenses/sampling+/1.0,169000000,151000000,27300,22900,274000,57300,34700,168000000,123000,62800,1310,169000000,200000
+https://creativecommons.org/licenses/by-nc-sa/2.5,31600000,22800000,20100,5400,83100,15200,10400,29900000,258000,250000,1570,31600000,7290
+https://creativecommons.org/licenses/nc-sa/1.0,24100000,17500000,10400,3820,58800,11800,10600,23300000,67500,280000,1200,24100000,42400

data/2024Q2/README.md

@@ -0,0 +1,20 @@
+# 2024Q2 Quantifying the Commons
+<!-- GCS Start -->
+## Data Source: Google Custom Search
+
+<!-- Country Report Start -->
+### Country Report
+![Number of Google Webpages Licensed by Country](3-report/gcs_country_report.png)
+Number of Google Webpages Licensed by Country
+<!-- Country Report End -->
+<!-- License Type Report Start -->
+### License Type Report
+![Number of Webpages Licensed by License Type](3-report/gcs_licensetype_report.png)
+Number of Webpages Licensed by License Type
+<!-- License Type Report End -->
+<!-- Language Report Start -->
+### Language Report
+![Number of Google Webpages Licensed by Language](3-report/gcs_language_report.png)
+Number of Google Webpages Licensed by Language
+<!-- Language Report End -->
+<!-- GCS End -->

data/2024Q2/state.yaml

@@ -0,0 +1 @@
+total_records_retrieved: 273

data/2024Q3/1-fetch/flickr_fetched/rec.txt

@@ -0,0 +1 @@
+1 1 1

data/2024Q3/1-fetch/gcs_fetched.csv

@@ -0,0 +1,18 @@
+LICENSE TYPE, No Priori, United States, Canada, India, United Kingdom, Australia, Japan, English, Spanish, French, Arabic, Chinese (Simplified), Indonesian
+https://creativecommons.org/licenses/by/2.5,235000000,208000000,112000,82400,406000,701000,76800,225000000,3940000,835000,88300,235000000,115000
+https://creativecommons.org/licenses/by/4.0,412000000,334000000,702000,360000,7250000,770000,675000,381000000,16100000,5260000,255000,412000000,11900000
+https://creativecommons.org/licenses/by-nc-sa/2.5,31400000,22700000,21200,5400,83000,15000,10300,29800000,254000,248000,1580,31400000,7480
+https://creativecommons.org/licenses/nc/1.0,54600000,43400000,25100,8390,128000,22200,30500,53600000,136000,479000,3760,54500000,81500
+https://creativecommons.org/licenses/by-nc-sa/1.0,24000000,17400000,11300,3810,59100,11900,10600,23300000,66900,276000,1220,24000000,42900
+https://creativecommons.org/licenses/by/2.1,219000000,189000000,132000,56200,477000,139000,269000,211000000,494000,232000,19800,219000000,119000
+https://creativecommons.org/licenses/nc-sampling+/1.0,41200000,32000000,12900,4590,82300,14600,18800,41000000,52800,37800,381,41200000,31100
+https://creativecommons.org/licenses/by-sa/4.0,175000000,140000000,126000,79300,341000,103000,123000,158000000,6800000,3860000,81300,175000000,626000
+https://creativecommons.org/licenses/by/2.1,217000000,189000000,120000,55500,456000,134000,278000,209000000,219000,241000,20300,217000000,116000
+https://creativecommons.org/licenses/by-nc-nd/2.1,66000000,51100000,21000,9200,144000,25100,30400,64300000,167000,581000,3690,66100000,19300
+https://creativecommons.org/licenses/by-nc/3.0,88200000,77800000,32000,14000,289000,91900,43000,85000000,7630000,6890000,13700,88300000,40200
+https://creativecommons.org/licenses/by/2.1,269000000,235000000,122000,54500,477000,124000,277000,261000000,455000,1900000,20100,269000000,113000
+https://creativecommons.org/licenses/by-nd/2.0,32500000,23600000,17000,6910,57500,19500,20100,31200000,121000,47200,5290,32500000,7660
+https://creativecommons.org/licenses/by-nc-sa/3.0,35800000,29800000,18000,5760,149000,69400,12200,33700000,623000,4440000,7890,35800000,16000
+https://creativecommons.org/licenses/sa/1.0,120000000,101000000,23500,16600,167000,56500,24400,116000000,247000,314000,29100,120000000,213000
+https://creativecommons.org/licenses/sa/1.0,120000000,101000000,23500,16600,167000,56500,24400,116000000,247000,314000,29100,120000000,213000
+https://creativecommons.org/licenses/by-nc-sa/2.0,20400000,13800000,21400,3730,65000,21700,9270,19100000,92600,773000,4470,20400000,5740

data/2024Q3/README.md

@@ -0,0 +1,19 @@
+# 2024Q3 Quantifying the Commons
+## Data Source: Google Custom Search
+<!-- Google Custom Search Start -->
+<!-- Country Report Start -->
+### Country Report
+![Number of Google Webpages Licensed by Country](3-report/gcs_country_report.png)
+Number of Google Webpages Licensed by Country
+<!-- Country Report End -->
+<!-- License Type Report Start -->
+### License Type Report
+![Number of Webpages Licensed by License Type](3-report/gcs_licensetype_report.png)
+Number of Webpages Licensed by License Type
+<!-- License Type Report End -->
+<!-- Language Report Start -->
+### Language Report
+![Number of Google Webpages Licensed by Language](3-report/gcs_language_report.png)
+Number of Google Webpages Licensed by Language
+<!-- Language Report End -->
+<!-- Google Custom Search End -->

data/2024Q3/state.yaml

@@ -0,0 +1 @@
+total_records_retrieved: 390