update to Python 3.11 (#769)

* upgrade to 3.11

* fix?

* fix shape

* prep release

* prep for updated client release

* update redis

* remove copy

* add build

* add twine

* fix manifest

* minor updates from release

* replace document ready with vanilla js

(cherry picked from commit 180a240)

* hawcprd -> hawc

(cherry picked from commit 728f577)

* fix admin export and write test

* fix template tag

* fix strange apis

* remove old endpoint tag list

* fix api issues

* fix errors w/ invalid user input

* fix view error

* fix another regression from #732

* fix deprecations

* remove extra venv

* update docs

.github/workflows/main.yml

@@ -33,7 +33,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v3
       with:
-        python-version: '3.9'
+        python-version: '3.11'
         architecture: 'x64'
     - name: try to restore pip cache
       uses: actions/cache@v3
@@ -149,7 +149,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v3
       with:
-        python-version: '3.9'
+        python-version: '3.11'
         architecture: 'x64'
     - name: try to restore pip cache
       uses: actions/cache@v3

.readthedocs.yml

@@ -1,9 +1,9 @@
 # https://docs.readthedocs.io/en/stable/config-file/v2.html
 version: 2
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.9"
+    python: "3.11"
 mkdocs:
   configuration: docs/mkdocs.yml
 python:

compose/app/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim
+FROM python:3.11-slim
 
 ENV PYTHONUNBUFFERED 1
 

docs/docs/development.md

@@ -5,7 +5,7 @@ Assessment Workspace Collaborative project.  To begin you should have the
 following applications installed on your local development system:
 
 - [Git](https://git-scm.com/)
-- [Python](https://www.python.org/) = 3.9
+- [Python](https://www.python.org/) == 3.11
 - [Node.js](https://nodejs.org)
 - [Yarn](https://yarnpkg.com/) < 2
 - [PostgreSQL](https://www.postgresql.org/) >= 12
@@ -50,7 +50,7 @@ For Windows, use anaconda or miniconda to get requirements can be used to get de
 :: create a conda environment with our hard to get dependencies
 conda create --name hawc
 conda activate hawc
-conda install python=3.9 postgresql
+conda install python=3.11 postgresql
 conda install -c conda-forge nodejs
 conda install -c conda-forge yarn=1.22.19
 

hawc/apps/common/renderers.py

@@ -102,7 +102,7 @@ class PandasCsvRenderer(PandasBaseRenderer):
 
     def render_dataframe(self, export: FlatExport, response: Response) -> str:
         # set line terminator to keep consistent on windows too
-        return export.df.to_csv(index=False, line_terminator="\n")
+        return export.df.to_csv(index=False, lineterminator="\n")
 
 
 class PandasTsvRenderer(PandasBaseRenderer):
@@ -115,7 +115,7 @@ class PandasTsvRenderer(PandasBaseRenderer):
 
     def render_dataframe(self, export: FlatExport, response: Response) -> str:
         # set line terminator to keep consistent on windows too
-        return export.df.to_csv(index=False, sep="\t", line_terminator="\n")
+        return export.df.to_csv(index=False, sep="\t", lineterminator="\n")
 
 
 class PandasJsonRenderer(PandasBaseRenderer):

hawc/refml/topics.py

@@ -73,7 +73,6 @@ def topic_model_tsne(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]:
     logger.info(f"Training topic model with {df.shape[0]} references")
 
     df.loc[:, "text"] = df.text.apply(stem_and_tokenize)
-    tfidf_transformer = TfidfTransformer()
     vectorizer = CountVectorizer(analyzer="word", max_features=10000)
     x_counts = vectorizer.fit_transform(df.text)
     tfidf_transformer = TfidfTransformer()

requirements/base.txt

@@ -14,29 +14,29 @@ djangorestframework==3.13.1
 gunicorn==20.1.0
 
 # tasks
-celery==5.2.3
+celery==5.2.7
   billiard==3.6.4.0
-  kombu==5.2.3
+  kombu==5.2.4
 
 # utilities
 jsonschema==4.4.0
-lxml==4.9.1
-psycopg2-binary==2.9.3
+lxml==4.9.2
+psycopg2-binary==2.9.5
 redis==4.5.4
 requests==2.27.1
 pydantic==1.9.0
 myst-parser==0.16.1
 bleach[css]==5.0.1
 
 # computational
-numpy==1.22.2
-pandas==1.4.3
+numpy==1.24.1
+pandas==1.5.2
   openpyxl==3.0.10
 plotly==5.6.0
-pyarrow==7.0.0
-scipy==1.8.0
+pyarrow==10.0.1
+scipy==1.10.0
 nltk==3.7
-scikit-learn==1.0.2
+scikit-learn==1.2.2
 bmds==22.1
 
 # exports

requirements/dev.txt

@@ -17,7 +17,7 @@ django-extensions==3.1.5
 faker==13.3.0
 
 # tests
-coverage==6.3.2
+coverage==7.2.3
 pytest==7.0.1
 pytest-django==4.5.2
 vcrpy==4.1.1

setup.cfg

@@ -10,12 +10,12 @@ author_email = shapiromatron@gmail.com
 license = MIT
 classifiers =
     Programming Language :: Python
-    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.11
 
 [options]
 include_package_data = true
 packages = hawc
-python_requires = >=3.9
+python_requires = >=3.11
 
 [options.entry_points]
 console_scripts =

tests/hawc/apps/lit/test_managers.py

@@ -9,10 +9,10 @@ def test_as_dataframe(self, db_keys):
         # assert empty tag list works
         df = models.ReferenceTags.objects.as_dataframe(db_keys.assessment_working)
         assert df.shape == (0, 2)
-        assert df.to_csv(index=False, line_terminator="\n") == "reference_id,tag_id\n"
+        assert df.to_csv(index=False, lineterminator="\n") == "reference_id,tag_id\n"
 
         # add a tag and make sure it appears
         models.Reference.objects.get(id=1).tags.set([2])
         df = models.ReferenceTags.objects.as_dataframe(db_keys.assessment_working)
         assert df.shape == (1, 2)
-        assert df.to_csv(index=False, line_terminator="\n") == "reference_id,tag_id\n1,2\n"
+        assert df.to_csv(index=False, lineterminator="\n") == "reference_id,tag_id\n1,2\n"

tests/hawc/apps/lit/test_serializers.py

@@ -88,7 +88,7 @@ def test_BulkReferenceTagSerializer(db_keys):
     assert serializer.is_valid() is True
     serializer.bulk_create_tags()
     resp = ReferenceTags.objects.as_dataframe(db_keys.assessment_working).to_csv(
-        index=False, line_terminator="\n"
+        index=False, lineterminator="\n"
     )
     assert resp == "reference_id,tag_id\n1,4\n"
 
@@ -100,7 +100,7 @@ def test_BulkReferenceTagSerializer(db_keys):
 
     reference.refresh_from_db()
     resp = ReferenceTags.objects.as_dataframe(db_keys.assessment_working).to_csv(
-        index=False, line_terminator="\n"
+        index=False, lineterminator="\n"
     )
     assert resp == "reference_id,tag_id\n1,3\n"
 
@@ -111,7 +111,7 @@ def test_BulkReferenceTagSerializer(db_keys):
     assert serializer.is_valid() is True
     serializer.bulk_create_tags()
     resp = ReferenceTags.objects.as_dataframe(db_keys.assessment_working).to_csv(
-        index=False, line_terminator="\n"
+        index=False, lineterminator="\n"
     )
     assert resp == csv
 

tests/hawc/refml/test_topics.py

@@ -54,12 +54,23 @@ def test_success(self, load_nltk_data):
                 "The Lobster Quadrille",
                 "Who Stole the Tarts?",
                 "Alice’s Evidence",
+                "The Looking-Glass House",
+                "The Garden of Live Flowers",
+                "Looking-Glass Insects",
+                "Tweedledum and Tweedledee",
+                "Jam Every Other Day",
+                "Humpty Dumpty",
+                "The Lion and the Unicorn",
+                "The Great Art of Riding",
+                "Queen Alice",
+                "Shaking and Waking",
+                "Who Dreamed It?",
             ],
             name="text",
         ).to_frame()
         df, df_topics = topics.topic_model_tsne(input_df)
 
-        assert df.shape == (12, 3)
+        assert df.shape == (23, 3)
         assert set(df.columns.tolist()) == {"max_topic", "tsne_x", "tsne_y"}
 
         assert df_topics.shape == (10, 2)