From cefcb0b82d104d6e7cce482a43563aae5b507182 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Wed, 5 Oct 2022 10:48:35 +0200 Subject: [PATCH 01/16] setup image and add readme --- Dockerfile | 2 +- README.md | 24 ++++++++++++++++++++++++ requirements.txt | 28 ++++++++++++++-------------- 3 files changed, 39 insertions(+), 15 deletions(-) create mode 100644 README.md diff --git a/Dockerfile b/Dockerfile index e23b7eb..023c2be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.6-slim-buster +FROM python:3.10.6-slim-buster COPY requirements.txt /requirements.txt RUN pip install -r /requirements.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..9287fe3 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ + +# release + +- create `py-${PYTHON_VERSION}-${YEAR}-${QUARTER}` branch +- update `Dockerfile` source image python version +- update `requirements.txt` from [data-setup specs/releases/glovebox.txt](https://github.com/lewagon/data-setup/blob/master/specs/releases/glovebox.txt) +- push branch and draft a new release, which triggers the `ghcr_publish_on_release_tag` GHA + +# usage + +``` bash +gh release list # list releases +``` + +# files + +``` bash +.github/workflows/ghcr_publish_on_release_tag.yml # publication GHA +.github/workflows/ghcr_build_latest_on_push.yml # useless (no push/merge on master) +``` + +# source + +requirements.txt [data-setup specs/releases/past/2022_Q4/glovebox.txt](https://github.com/lewagon/data-setup/blob/setup-2022-q4/specs/releases/past/2022_Q4/glovebox.txt) diff --git a/requirements.txt b/requirements.txt index ca88bd5..2016adc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ -pytest -pylint -ipdb -pyyaml -nbresult -autopep8 -flake8 -lxml -requests -bs4 -pandas==1.2 -matplotlib -numpy -scikit-learn +pytest==7.1.3 +pylint==2.15.3 +nbresult==0.0.8 +autopep8==1.7.0 +flake8==5.0.4 +ipdb==0.13.9 +requests==2.28.1 +beautifulsoup4==4.11.1 +PyYAML==6.0 +lxml==4.9.1 +numpy==1.23.3 +pandas==1.5.0 +matplotlib==3.6.0 +scikit-learn==1.1.2 From 4e5daab44814aded1e958d87d6d8758dbfcdc0b7 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Wed, 5 Oct 2022 10:53:50 +0200 Subject: [PATCH 02/16] update readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 9287fe3..f292027 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,14 @@ - update `requirements.txt` from [data-setup specs/releases/glovebox.txt](https://github.com/lewagon/data-setup/blob/master/specs/releases/glovebox.txt) - push branch and draft a new release, which triggers the `ghcr_publish_on_release_tag` GHA +# draft a release + +[draft a new release](https://github.com/lewagon/data-runner/releases): +- choose a tag: use branch name (ignore branch/tag naming confusion warning) / create +- release title: use branch name +- description: ... +- publish release + # usage ``` bash From a8915305090a0ef960db811f477dc61012eb534c Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Wed, 5 Oct 2022 11:02:56 +0200 Subject: [PATCH 03/16] update readme --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f292027..1bd1e60 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,9 @@ - update `Dockerfile` source image python version - update `requirements.txt` from [data-setup specs/releases/glovebox.txt](https://github.com/lewagon/data-setup/blob/master/specs/releases/glovebox.txt) - push branch and draft a new release, which triggers the `ghcr_publish_on_release_tag` GHA +- update `data-solutions` glovebox image -# draft a release +## draft a release [draft a new release](https://github.com/lewagon/data-runner/releases): - choose a tag: use branch name (ignore branch/tag naming confusion warning) / create @@ -14,6 +15,11 @@ - description: ... - publish release +## update data-solutions glovebox image + +[edit data-solutions/test-solutions-action/Dockerfile](https://github.com/lewagon/data-solutions/blob/master/test-solutions-action/Dockerfile): +- replace `FROM ghcr.io/lewagon/data-runner:$TAG_NAME` + # usage ``` bash From 1de9866bb12acca1e3eaa977c87e7a959745fe29 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Wed, 5 Oct 2022 12:02:27 +0200 Subject: [PATCH 04/16] update readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 1bd1e60..cf999dd 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,12 @@ - description: ... - publish release +## list container images + +``` bash +gh release list --repo lewagon/data-runner +``` + ## update data-solutions glovebox image [edit data-solutions/test-solutions-action/Dockerfile](https://github.com/lewagon/data-solutions/blob/master/test-solutions-action/Dockerfile): From d59367be3eac8c4f0db6c9c32f3e1c7e8d57bcd9 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Wed, 5 Oct 2022 12:06:00 +0200 Subject: [PATCH 05/16] update readme --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index cf999dd..00e4b0d 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,6 @@ gh release list --repo lewagon/data-runner [edit data-solutions/test-solutions-action/Dockerfile](https://github.com/lewagon/data-solutions/blob/master/test-solutions-action/Dockerfile): - replace `FROM ghcr.io/lewagon/data-runner:$TAG_NAME` -# usage - -``` bash -gh release list # list releases -``` - # files ``` bash From c5d6110cc3e24575d93904c11a54ba1621df422e Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Wed, 5 Oct 2022 15:40:26 +0200 Subject: [PATCH 06/16] update readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 00e4b0d..2e82a88 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ - update `requirements.txt` from [data-setup specs/releases/glovebox.txt](https://github.com/lewagon/data-setup/blob/master/specs/releases/glovebox.txt) - push branch and draft a new release, which triggers the `ghcr_publish_on_release_tag` GHA - update `data-solutions` glovebox image +- update `active_admin` programs to use the image ## draft a release @@ -26,6 +27,12 @@ gh release list --repo lewagon/data-runner [edit data-solutions/test-solutions-action/Dockerfile](https://github.com/lewagon/data-solutions/blob/master/test-solutions-action/Dockerfile): - replace `FROM ghcr.io/lewagon/data-runner:$TAG_NAME` +## update `active_admin` programs to use the image + +[edit data program](https://kitt.lewagon.com/active_admin/programs/10/edit) +- runner tag: use branch name +- update program + # files ``` bash From d65a1eb032f17ec59cd612e45e8bff57af5794d8 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Wed, 5 Oct 2022 15:42:58 +0200 Subject: [PATCH 07/16] update readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2e82a88..8ac55cb 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,12 @@ - create `py-${PYTHON_VERSION}-${YEAR}-${QUARTER}` branch - update `Dockerfile` source image python version - update `requirements.txt` from [data-setup specs/releases/glovebox.txt](https://github.com/lewagon/data-setup/blob/master/specs/releases/glovebox.txt) -- push branch and draft a new release, which triggers the `ghcr_publish_on_release_tag` GHA +- push branch +- draft a `release` (which triggers the `ghcr_publish_on_release_tag` GHA) - update `data-solutions` glovebox image - update `active_admin` programs to use the image -## draft a release +## draft a `release` [draft a new release](https://github.com/lewagon/data-runner/releases): - choose a tag: use branch name (ignore branch/tag naming confusion warning) / create @@ -16,13 +17,13 @@ - description: ... - publish release -## list container images +list container images: ``` bash gh release list --repo lewagon/data-runner ``` -## update data-solutions glovebox image +## update `data-solutions` glovebox image [edit data-solutions/test-solutions-action/Dockerfile](https://github.com/lewagon/data-solutions/blob/master/test-solutions-action/Dockerfile): - replace `FROM ghcr.io/lewagon/data-runner:$TAG_NAME` From ae665e05d743558a706498d43ed1be6491ead339 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Wed, 5 Oct 2022 15:51:19 +0200 Subject: [PATCH 08/16] update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8ac55cb..ebb9e64 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ ## draft a `release` [draft a new release](https://github.com/lewagon/data-runner/releases): -- choose a tag: use branch name (ignore branch/tag naming confusion warning) / create -- release title: use branch name +- choose a tag: use `${BRANCH_NAME}-v1` (ignore branch/tag naming confusion warning) / create +- release title: use `${BRANCH_NAME}-v1` - description: ... - publish release @@ -31,7 +31,7 @@ gh release list --repo lewagon/data-runner ## update `active_admin` programs to use the image [edit data program](https://kitt.lewagon.com/active_admin/programs/10/edit) -- runner tag: use branch name +- runner tag: use `${BRANCH_NAME}-v1` - update program # files From e9f3fb9fb294a3c9953e9c72443e476e353ee2ca Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Wed, 5 Oct 2022 15:52:54 +0200 Subject: [PATCH 09/16] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ebb9e64..be7a19c 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ [draft a new release](https://github.com/lewagon/data-runner/releases): - choose a tag: use `${BRANCH_NAME}-v1` (ignore branch/tag naming confusion warning) / create +- choose a target: use `${BRANCH_NAME}` - release title: use `${BRANCH_NAME}-v1` - description: ... - publish release From 3f70989cf40e197489f3fed78190ded889ea1b1e Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Thu, 6 Oct 2022 10:19:00 +0200 Subject: [PATCH 10/16] add make required by the student tests --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 023c2be..a6abc69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,9 @@ FROM python:3.10.6-slim-buster + +# data-solutions gha works without this, but the glovebox code validation fails +RUN apt-get update && apt-get install -y --no-install-recommends make curl + COPY requirements.txt /requirements.txt RUN pip install -r /requirements.txt -WORKDIR /data-challenges +WORKDIR /workspace From 4eefb869a1ccb23517382d9fe9ea0122d756e862 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Thu, 6 Oct 2022 10:24:49 +0200 Subject: [PATCH 11/16] update readme --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index be7a19c..566b563 100644 --- a/README.md +++ b/README.md @@ -35,11 +35,26 @@ gh release list --repo lewagon/data-runner - runner tag: use `${BRANCH_NAME}-v1` - update program +# release update + +- push branch +- edit `release` (which triggers the `ghcr_publish_on_release_tag` GHA) +- update `data-solutions` glovebox image +- update `active_admin` programs to use the image + +## edit `release` + +[edit release](https://github.com/lewagon/data-runner/releases): +- edit the release +- choose a tag: `${BRANCH_NAME}-v2` / create +- release title: `${BRANCH_NAME}-v2` +- update release + # files ``` bash .github/workflows/ghcr_publish_on_release_tag.yml # publication GHA -.github/workflows/ghcr_build_latest_on_push.yml # useless (no push/merge on master) +.github/workflows/ghcr_build_latest_on_push.yml # unused ? (no push/merge on master) ``` # source From 2b3eb4fb4439170ed8a20213d35916fbc1156a57 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Fri, 7 Oct 2022 10:44:43 +0200 Subject: [PATCH 12/16] add missing package --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2016adc..377d378 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ lxml==4.9.1 numpy==1.23.3 pandas==1.5.0 matplotlib==3.6.0 +memoized-property==1.0.3 scikit-learn==1.1.2 From b0fbea14e760c6d4737b7ba47419008ef5d36c3b Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Fri, 14 Oct 2022 10:27:54 +0200 Subject: [PATCH 13/16] update requirements according to latest setup with pandas 1.4.4 --- requirements.txt | 99 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 377d378..e0f218f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,94 @@ -pytest==7.1.3 -pylint==2.15.3 -nbresult==0.0.8 +astroid==2.12.11 +asttokens==2.0.8 +attrs==22.1.0 autopep8==1.7.0 +backcall==0.2.0 +beautifulsoup4==4.11.1 +CacheControl==0.12.11 +cachy==0.3.0 +certifi==2022.9.24 +cffi==1.15.1 +charset-normalizer==2.1.1 +cleo==1.0.0a5 +crashtest==0.3.1 +cryptography==38.0.1 +cycler==0.11.0 +decorator==5.1.1 +dill==0.3.5.1 +distlib==0.3.6 +dulwich==0.20.46 +executing==1.1.1 +filelock==3.8.0 flake8==5.0.4 +fonttools==4.37.4 +html5lib==1.1 +idna==3.4 +iniconfig==1.1.1 ipdb==0.13.9 -requests==2.28.1 -beautifulsoup4==4.11.1 -PyYAML==6.0 +ipython==8.5.0 +isort==5.10.1 +jaraco.classes==3.2.3 +jedi==0.18.1 +jeepney==0.8.0 +joblib==1.1.1 +jsonschema==4.16.0 +keyring==23.9.3 +kiwisolver==1.4.4 +lazy-object-proxy==1.7.1 +lockfile==0.12.2 lxml==4.9.1 -numpy==1.23.3 -pandas==1.5.0 -matplotlib==3.6.0 -memoized-property==1.0.3 +matplotlib==3.5.3 +matplotlib-inline==0.1.6 +mccabe==0.7.0 +more-itertools==8.14.0 +msgpack==1.0.4 +nbresult==0.0.8 +numpy==1.23.4 +packaging==21.3 +pandas==1.4.4 +parso==0.8.3 +pexpect==4.8.0 +pickleshare==0.7.5 +Pillow==9.2.0 +pkginfo==1.8.3 +platformdirs==2.5.2 +pluggy==1.0.0 +poetry==1.2.2 +poetry-core==1.3.2 +poetry-plugin-export==1.1.2 +prompt-toolkit==3.0.31 +ptyprocess==0.7.0 +pure-eval==0.2.2 +py==1.11.0 +pycodestyle==2.9.1 +pycparser==2.21 +pyflakes==2.5.0 +Pygments==2.13.0 +pylev==1.4.0 +pylint==2.15.4 +pyparsing==3.0.9 +pyrsistent==0.18.1 +pytest==7.1.3 +python-dateutil==2.8.2 +pytz==2022.4 +PyYAML==6.0 +requests==2.28.1 +requests-toolbelt==0.9.1 scikit-learn==1.1.2 +scipy==1.9.2 +SecretStorage==3.3.3 +setuptools-scm==6.4.2 +shellingham==1.5.0 +six==1.16.0 +soupsieve==2.3.2.post1 +stack-data==0.5.1 +threadpoolctl==3.1.0 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.11.5 +traitlets==5.4.0 +urllib3==1.26.12 +virtualenv==20.16.5 +wcwidth==0.2.5 +webencodings==0.5.1 +wrapt==1.14.1 From f01eafe0163b45173db9ca60f5f59f2a9e235e02 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Fri, 14 Oct 2022 11:18:18 +0200 Subject: [PATCH 14/16] add memoized properties to glovebox --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index e0f218f..c00bcf8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,6 +40,7 @@ lxml==4.9.1 matplotlib==3.5.3 matplotlib-inline==0.1.6 mccabe==0.7.0 +memoized-property==1.0.3 more-itertools==8.14.0 msgpack==1.0.4 nbresult==0.0.8 From 93829353e43510a8831224c6e41084fef2ffa740 Mon Sep 17 00:00:00 2001 From: Gaetan Manchon Date: Mon, 24 Oct 2022 18:26:28 +0200 Subject: [PATCH 15/16] update dependencies --- requirements.txt | 57 ++++++++++++------------------------------------ 1 file changed, 14 insertions(+), 43 deletions(-) diff --git a/requirements.txt b/requirements.txt index c00bcf8..4d4c913 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,48 +1,31 @@ -astroid==2.12.11 +astroid==2.11.7 asttokens==2.0.8 attrs==22.1.0 -autopep8==1.7.0 +autopep8==1.6.0 backcall==0.2.0 beautifulsoup4==4.11.1 -CacheControl==0.12.11 -cachy==0.3.0 certifi==2022.9.24 -cffi==1.15.1 charset-normalizer==2.1.1 -cleo==1.0.0a5 -crashtest==0.3.1 -cryptography==38.0.1 cycler==0.11.0 decorator==5.1.1 -dill==0.3.5.1 -distlib==0.3.6 -dulwich==0.20.46 +dill==0.3.6 executing==1.1.1 -filelock==3.8.0 -flake8==5.0.4 -fonttools==4.37.4 -html5lib==1.1 +flake8==4.0.1 +fonttools==4.38.0 idna==3.4 iniconfig==1.1.1 ipdb==0.13.9 ipython==8.5.0 isort==5.10.1 -jaraco.classes==3.2.3 jedi==0.18.1 -jeepney==0.8.0 joblib==1.1.1 -jsonschema==4.16.0 -keyring==23.9.3 kiwisolver==1.4.4 lazy-object-proxy==1.7.1 -lockfile==0.12.2 lxml==4.9.1 matplotlib==3.5.3 matplotlib-inline==0.1.6 -mccabe==0.7.0 +mccabe==0.6.1 memoized-property==1.0.3 -more-itertools==8.14.0 -msgpack==1.0.4 nbresult==0.0.8 numpy==1.23.4 packaging==21.3 @@ -50,36 +33,26 @@ pandas==1.4.4 parso==0.8.3 pexpect==4.8.0 pickleshare==0.7.5 -Pillow==9.2.0 -pkginfo==1.8.3 +Pillow==9.1.1 platformdirs==2.5.2 pluggy==1.0.0 -poetry==1.2.2 -poetry-core==1.3.2 -poetry-plugin-export==1.1.2 prompt-toolkit==3.0.31 ptyprocess==0.7.0 pure-eval==0.2.2 py==1.11.0 -pycodestyle==2.9.1 -pycparser==2.21 -pyflakes==2.5.0 +pycodestyle==2.8.0 +pyflakes==2.4.0 Pygments==2.13.0 -pylev==1.4.0 -pylint==2.15.4 +pylint==2.14.5 pyparsing==3.0.9 -pyrsistent==0.18.1 pytest==7.1.3 python-dateutil==2.8.2 -pytz==2022.4 -PyYAML==6.0 +pytz==2022.1 +PyYAML==5.4.1 requests==2.28.1 -requests-toolbelt==0.9.1 scikit-learn==1.1.2 -scipy==1.9.2 -SecretStorage==3.3.3 +scipy==1.8.1 setuptools-scm==6.4.2 -shellingham==1.5.0 six==1.16.0 soupsieve==2.3.2.post1 stack-data==0.5.1 @@ -87,9 +60,7 @@ threadpoolctl==3.1.0 toml==0.10.2 tomli==2.0.1 tomlkit==0.11.5 -traitlets==5.4.0 +traitlets==5.5.0 urllib3==1.26.12 -virtualenv==20.16.5 wcwidth==0.2.5 -webencodings==0.5.1 wrapt==1.14.1 From 4fa4411dbe6898ae6e6c0cb70aed0a87f228f0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Manchon?= Date: Thu, 6 Jul 2023 18:45:07 +0200 Subject: [PATCH 16/16] add missing unidecode for lewagon-assess/data-paintings-denoising-lyrics-detector-solution --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 4d4c913..4a6c415 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +appnope==0.1.3 astroid==2.11.7 asttokens==2.0.8 attrs==22.1.0 @@ -61,6 +62,7 @@ toml==0.10.2 tomli==2.0.1 tomlkit==0.11.5 traitlets==5.5.0 +Unidecode==1.3.6 urllib3==1.26.12 wcwidth==0.2.5 wrapt==1.14.1