From 82aa5d4dcbe676b121dc931b0afa09a82de8e3d7 Mon Sep 17 00:00:00 2001 From: konstin Date: Wed, 27 Sep 2023 22:26:36 +0200 Subject: [PATCH] Fix macos releases --- .github/workflows/release.yml | 2 +- pypi_analysis/bigquery.ipynb | 255 -------------------------- pypi_analysis/bigquery_entirety.ipynb | 107 ----------- pypi_analysis/download.py | 38 ---- 4 files changed, 1 insertion(+), 401 deletions(-) delete mode 100644 pypi_analysis/bigquery.ipynb delete mode 100644 pypi_analysis/bigquery_entirety.ipynb delete mode 100644 pypi_analysis/download.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4ef388..bbff151 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: - name: Build wheels - universal2 uses: PyO3/maturin-action@v1 with: - args: --release --strip --universal2 --out dist + args: --release --strip --target --target universal2-apple-darwin --out dist env: RUSTFLAGS: "-C link-arg=-undefined -C link-arg=dynamic_lookup" - name: Upload wheels diff --git a/pypi_analysis/bigquery.ipynb b/pypi_analysis/bigquery.ipynb deleted file mode 100644 index 064dc90..0000000 --- a/pypi_analysis/bigquery.ipynb +++ /dev/null @@ -1,255 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# big query datasets PEP 508 analysis\n", - "\n", - "We analyze the entirety of all `requires_dist` in pypi's big query dataset from mid december 2022" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "ExecuteTime": { - "start_time": "2023-03-07T17:16:20.344005Z", - "end_time": "2023-03-07T17:16:20.351475Z" - } - }, - "outputs": [], - "source": [ - "import json\n", - "from pathlib import Path\n", - "\n", - "import packaging.requirements\n", - "from pep508_rs import Requirement, Pep508Error, MarkerEnvironment\n", - "from tqdm import tqdm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "ExecuteTime": { - "start_time": "2023-03-07T17:16:20.355679Z", - "end_time": "2023-03-07T17:18:52.791632Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|█████████▉| 8175705/8175706 [02:28<00:00, 54923.32it/s] \n" - ] - } - ], - "source": [ - "env = MarkerEnvironment.current()\n", - "\n", - "packages = 0\n", - "valid = 0\n", - "invalid = 0\n", - "\n", - "general_parse_errors = list()\n", - "rs_parse_errors = list()\n", - "all_warnings = list()\n", - "\n", - "\n", - "num_lines = sum(1 for line in Path(\"pipy_requires_dist.ndjson\").open(\"r\"))\n", - "with Path(\"pipy_requires_dist.ndjson\").open(\"r\") as fp:\n", - " for line in tqdm(fp, total=num_lines):\n", - " num_lines -= 1\n", - " if num_lines == 0:\n", - " break\n", - " packages += 1\n", - " data = json.loads(line)\n", - " all_valid = True\n", - " for requirement in data[\"requires_dist\"]:\n", - " try:\n", - " requirement = Requirement(requirement)\n", - " except Pep508Error as e:\n", - " try:\n", - " packaging.requirements.Requirement(requirement)\n", - " if \"===\" in requirement:\n", - " general_parse_errors.append(e)\n", - " else:\n", - " rs_parse_errors.append(e)\n", - " except packaging.requirements.InvalidRequirement as e:\n", - " general_parse_errors.append(e)\n", - " all_valid = False\n", - " continue\n", - " # print(data[\"name\"], data[\"version\"], requirement, e)\n", - " result, warnings = requirement.evaluate_markers_and_report(env, [])\n", - " if warnings:\n", - " all_warnings.extend(warnings)\n", - " # print(warnings)\n", - " if all_valid:\n", - " valid += 1\n", - " else:\n", - " invalid += 1" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "ExecuteTime": { - "start_time": "2023-03-07T17:18:52.791385Z", - "end_time": "2023-03-07T17:18:52.792319Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "packages: 8175705 valid: 8152056 invalid: 23649 fraction invalid: 0.29%\n" - ] - } - ], - "source": [ - "# TODO: check what validation pypi currently does\n", - "# obviously they can't change old metadata though\n", - "print(\n", - " f\"packages: {packages} valid: {valid} invalid: {invalid} fraction invalid: {invalid / (valid + invalid):.2%}\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "ExecuteTime": { - "start_time": "2023-03-07T17:18:52.791533Z", - "end_time": "2023-03-07T17:18:52.793991Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(MarkerWarningKind.DeprecatedMarkerName, 'os.name is deprecated in favor of os_name', \"os_name == 'posix'\")\n", - "(MarkerWarningKind.DeprecatedMarkerName, 'platform.python_implementation is deprecated in favor of platform_python_implementation', \"platform_python_implementation != 'Jython'\")\n", - "(MarkerWarningKind.DeprecatedMarkerName, 'platform.python_implementation is deprecated in favor of platform_python_implementation', \"platform_python_implementation == 'Jython'\")\n", - "(MarkerWarningKind.DeprecatedMarkerName, 'sys.platform is deprecated in favor of sys_platform', \"sys_platform != 'win32'\")\n", - "(MarkerWarningKind.DeprecatedMarkerName, 'sys.platform is deprecated in favor of sys_platform', \"sys_platform == 'darwin'\")\n", - "(MarkerWarningKind.DeprecatedMarkerName, 'sys.platform is deprecated in favor of sys_platform', \"sys_platform == 'linux'\")\n", - "(MarkerWarningKind.DeprecatedMarkerName, 'sys.platform is deprecated in favor of sys_platform', \"sys_platform == 'win32'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 10.0 lexicographically', \"platform_release >= '10.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 11.0 lexicographically', \"platform_release >= '11.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 12.0 lexicographically', \"platform_release < '12.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 12.0 lexicographically', \"platform_release >= '12.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 13.0 lexicographically', \"platform_release < '13.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 13.0 lexicographically', \"platform_release >= '13.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 14.0 lexicographically', \"platform_release >= '14.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 15.0 lexicographically', \"platform_release >= '15.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 16.0 lexicographically', \"platform_release >= '16.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 17.0 lexicographically', \"platform_release >= '17.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 18.0 lexicographically', \"platform_release >= '18.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 19.0 lexicographically', \"platform_release >= '19.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 20.0 lexicographically', \"platform_release >= '20.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 21.0 lexicographically', \"platform_release >= '21.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 21.3 lexicographically', \"platform_release >= '21.3'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 21.4 lexicographically', \"platform_release >= '21.4'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 22.0 lexicographically', \"platform_release >= '22.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing 5.15.0-67-generic and 9.0 lexicographically', \"platform_release >= '9.0'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing x86_64 and armv0l lexicographically', \"platform_machine >= 'armv0l'\")\n", - "(MarkerWarningKind.LexicographicComparison, 'Comparing x86_64 and armv9l lexicographically', \"platform_machine <= 'armv9l'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version operator to compare python_version with '2.6', found 'in', evaluating to false\", \"python_version in '2.6'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version operator to compare python_version with '2.7', found 'in', evaluating to false\", \"python_version in '2.7'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_full_version, found '2.7.0 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.7.8 2.7.9 2.7.10 2.7.11 2.7.12 2.7.13 3.0.0 3.0.1 3.1.0 3.1.1 3.1.2 3.1.3 3.1.4 3.1.5 3.2.0 3.2.1 3.2.2 3.2.3 3.2.4 3.2.5 3.2.6 3.3.0 3.3.1 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6', evaluating to false: Version `2.7.0 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.7.8 2.7.9 2.7.10 2.7.11 2.7.12 2.7.13 3.0.0 3.0.1 3.1.0 3.1.1 3.1.2 3.1.3 3.1.4 3.1.5 3.2.0 3.2.1 3.2.2 3.2.3 3.2.4 3.2.5 3.2.6 3.3.0 3.3.1 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6` doesn't match PEP 440 rules\", \"python_full_version in '2.7.0 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.7.8 2.7.9 2.7.10 2.7.11 2.7.12 2.7.13 3.0.0 3.0.1 3.1.0 3.1.1 3.1.2 3.1.3 3.1.4 3.1.5 3.2.0 3.2.1 3.2.2 3.2.3 3.2.4 3.2.5 3.2.6 3.3.0 3.3.1 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_full_version, found '2.7.0 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.7.8', evaluating to false: Version `2.7.0 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.7.8` doesn't match PEP 440 rules\", \"python_full_version in '2.7.0 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.7.8'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_full_version, found '3.6.0, 3.6.1, 3.7.0, 3.7.1', evaluating to false: Version `3.6.0, 3.6.1, 3.7.0, 3.7.1` doesn't match PEP 440 rules\", \"python_full_version not in '3.6.0, 3.6.1, 3.7.0, 3.7.1'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_full_version, found '3.7.0, 3.7.1', evaluating to false: Version `3.7.0, 3.7.1` doesn't match PEP 440 rules\", \"python_full_version not in '3.7.0, 3.7.1'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '2,7', evaluating to false: Version `2,7` doesn't match PEP 440 rules\", \"python_version != '2,7'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '2.4, 2.5, 2.6', evaluating to false: Version `2.4, 2.5, 2.6` doesn't match PEP 440 rules\", \"python_version in '2.4, 2.5, 2.6'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '2.6 2.7 3.2', evaluating to false: Version `2.6 2.7 3.2` doesn't match PEP 440 rules\", \"python_version in '2.6 2.7 3.2'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '2.6 2.7 3.2 3.3', evaluating to false: Version `2.6 2.7 3.2 3.3` doesn't match PEP 440 rules\", \"python_version in '2.6 2.7 3.2 3.3'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '2.6 2.7', evaluating to false: Version `2.6 2.7` doesn't match PEP 440 rules\", \"python_version in '2.6 2.7'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '2.6, 2.7', evaluating to false: Version `2.6, 2.7` doesn't match PEP 440 rules\", \"python_version in '2.6, 2.7'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '2.6, 2.7, 3.2', evaluating to false: Version `2.6, 2.7, 3.2` doesn't match PEP 440 rules\", \"python_version in '2.6, 2.7, 3.2'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '2.6,2.7', evaluating to false: Version `2.6,2.7` doesn't match PEP 440 rules\", \"python_version in '2.6,2.7'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '2.7,3.3', evaluating to false: Version `2.7,3.3` doesn't match PEP 440 rules\", \"python_version in '2.7,3.3'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3,8', evaluating to false: Version `3,8` doesn't match PEP 440 rules\", \"python_version < '3,8'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.0, 3.1', evaluating to false: Version `3.0, 3.1` doesn't match PEP 440 rules\", \"python_version in '3.0, 3.1'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.0, 3.1', evaluating to false: Version `3.0, 3.1` doesn't match PEP 440 rules\", \"python_version not in '3.0, 3.1'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.0, 3.1, 3.2', evaluating to false: Version `3.0, 3.1, 3.2` doesn't match PEP 440 rules\", \"python_version not in '3.0, 3.1, 3.2'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.0, 3.1, 3.2, 3.3', evaluating to false: Version `3.0, 3.1, 3.2, 3.3` doesn't match PEP 440 rules\", \"python_version not in '3.0, 3.1, 3.2, 3.3'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.0, 3.1, 3.2, 3.3, 3.4', evaluating to false: Version `3.0, 3.1, 3.2, 3.3, 3.4` doesn't match PEP 440 rules\", \"python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.0, 3.1, 3.2, 3.3, 3.4, 3.5', evaluating to false: Version `3.0, 3.1, 3.2, 3.3, 3.4, 3.5` doesn't match PEP 440 rules\", \"python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6', evaluating to false: Version `3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6` doesn't match PEP 440 rules\", \"python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.2 3.3 3.4', evaluating to false: Version `3.2 3.3 3.4` doesn't match PEP 440 rules\", \"python_version in '3.2 3.3 3.4'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.3 3.4 3.5', evaluating to false: Version `3.3 3.4 3.5` doesn't match PEP 440 rules\", \"python_version in '3.3 3.4 3.5'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.4, <3.7', evaluating to false: Version `3.4, <3.7` doesn't match PEP 440 rules\", \"python_version >= '3.4, <3.7'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.7.*,3.9.*', evaluating to false: Version `3.7.*,3.9.*` doesn't match PEP 440 rules\", \"python_version != '3.7.*,3.9.*'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.8,3.8', evaluating to false: Version `3.8,3.8` doesn't match PEP 440 rules\", \"python_version in '3.8,3.8'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.8.', evaluating to false: Version `3.8.` doesn't match PEP 440 rules\", \"python_version < '3.8.'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.8.*,3.10.*', evaluating to false: Version `3.8.*,3.10.*` doesn't match PEP 440 rules\", \"python_version == '3.8.*,3.10.*'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found '3.9p', evaluating to false: Version `3.9p` doesn't match PEP 440 rules\", \"python_version < '3.9p'\")\n", - "(MarkerWarningKind.Pep440Error, \"Expected PEP 440 version to compare with python_version, found 'dev', evaluating to false: Version `dev` doesn't match PEP 440 rules\", \"python_version == 'dev'\")\n", - "(MarkerWarningKind.Pep440Error, 'Invalid operator/version combination: Operator <= must not be used in version ending with a star', \"python_version <= '3.5.*'\")\n" - ] - } - ], - "source": [ - "# Let's look at all the things that look unintentional\n", - "for i in sorted(set(all_warnings)):\n", - " print(i)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "ExecuteTime": { - "start_time": "2023-03-07T17:18:52.791937Z", - "end_time": "2023-03-07T17:18:52.886188Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "errors in both rust and python 37716\n", - "rust only errors 1\n", - "Expected a valid marker name, found 'python_implementation'\n", - "isal (>=0.3.0) ; sys_platform==\"linux\" and python_implementation != \"PyPy\"\n", - " ^^^^^^^^^^^^^^^^^^^^^\n" - ] - } - ], - "source": [ - "# Are there any cases our rust parser can't handle that the python parser (packaging.requirements) can handle?\n", - "print(\"errors in both rust and python\", len(general_parse_errors))\n", - "print(\"rust only errors\", len(rs_parse_errors))\n", - "print(*rs_parse_errors)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/pypi_analysis/bigquery_entirety.ipynb b/pypi_analysis/bigquery_entirety.ipynb deleted file mode 100644 index c3529be..0000000 --- a/pypi_analysis/bigquery_entirety.ipynb +++ /dev/null @@ -1,107 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "# NB: Unfinished prototype" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "from google.cloud import bigquery\n", - "\n", - "# noinspection PyTypeChecker\n", - "client = bigquery.Client(project=\"jupyter-local-project\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "table_id = \"the-psf.pypi.distribution_metadata\"\n", - "table = client.get_table(table_id)\n", - "table.schema" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "from tqdm import tqdm\n", - "\n", - "from datetime import datetime\n", - "import json\n", - "\n", - "\n", - "class DateTimeEncoder(json.JSONEncoder):\n", - " def default(self, o):\n", - " if isinstance(o, datetime):\n", - " return o.isoformat()\n", - "\n", - " return json.JSONEncoder.default(self, o)\n", - "\n", - "\n", - "# Save a lot of disk space\n", - "selected_fields = [field for field in table.schema if field.name != \"description\"]\n", - "\n", - "table_id = \"the-psf.pypi.distribution_metadata\"\n", - "table = client.get_table(table_id)\n", - "with Path(\"pypi_all.ndjson\").open(\"w\") as fp:\n", - " rows_iter = client.list_rows(table_id, selected_fields=selected_fields)\n", - " for row in tqdm(rows_iter, total=table.num_rows):\n", - " fp.write(json.dumps(dict(row), cls=DateTimeEncoder))\n", - " fp.write(\"\\n\")" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/pypi_analysis/download.py b/pypi_analysis/download.py deleted file mode 100644 index 6135a47..0000000 --- a/pypi_analysis/download.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -To run this: - * `pip install google-cloud-bigquery` - * Download https://cloud.google.com/sdk/docs/install - * Unpack, skip installation - * Create a free google cloud account - * `google-cloud-sdk/bin/gcloud auth application-default login` - * Check `$HOME/.config/gcloud/application_default_credentials.json` is there - * Run. Takes a while (15min - 1h) -""" - -import json -from pathlib import Path - -from google.cloud import bigquery -from tqdm import tqdm - - -def main(): - # noinspection PyTypeChecker - client = bigquery.Client(project="jupyter-local-project") - table_id = "the-psf.pypi.distribution_metadata" - table = client.get_table(table_id) - - selected_fields = [ - field - for field in table.schema - if field.name in ["name", "version", "filename", "requires_dist"] - ] - with Path("pipy_requires_dist.ndjson").open("w") as fp: - rows_iter = client.list_rows(table_id, selected_fields=selected_fields) - for row in tqdm(rows_iter, total=table.num_rows): - fp.write(json.dumps(dict(row))) - fp.write("\n") - - -if __name__ == "__main__": - main()