Update Python version to 3.13 in Dockerfile and Pipfile

.github/workflows/ci.yml

@@ -11,14 +11,24 @@ on:
       - master
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      python_version: ${{ steps.read_python_version.outputs.python_version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Read Python version
+        id: read_python_version
+        run: echo "::set-output name=python_version::$(cat .python-version)"
   black:
     runs-on: ubuntu-latest
+    needs: setup
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: ${{ needs.setup.outputs.python_version }}
       - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
@@ -37,12 +47,13 @@ jobs:
           python -m black --check .
   flake8:
     runs-on: ubuntu-latest
+    needs: setup
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: ${{ needs.setup.outputs.python_version }}
       - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
@@ -61,12 +72,13 @@ jobs:
           flake8
   mypy:
     runs-on: ubuntu-latest
+    needs: setup
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: ${{ needs.setup.outputs.python_version }}
       - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
@@ -84,6 +96,7 @@ jobs:
           python -m mypy .
   test:
     runs-on: ubuntu-latest
+    needs: setup
     services:
       mysql:
         image: mysql:8.0
@@ -104,13 +117,17 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: ${{ needs.setup.outputs.python_version }}
       - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
           restore-keys: |
             ${{ runner.os }}-pip-
+      - name: Install dependencies for building Python wheels (Ubuntu)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential python3-dev libffi-dev libssl-dev libpq-dev unixodbc-dev
       - name: Install pipenv
         run: |
           pip install pipenv
@@ -122,12 +139,17 @@ jobs:
         with:
           mongodb-version: 4.2
           mongodb-replica-set: heron_rs
-      - name: Create SQL Server testing database
-        run: |
-          python setup_sqlserver_test_db.py
       - name: Setup the test MLWH and Events databases
         run: |
+          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
+          curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
+          sudo apt-get update
+          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
+          sudo apt-get install -y unixodbc-dev
           python setup_test_db.py
+      - name: Create SQL Server testing database
+        run: |
+          python setup_sqlserver_test_db.py
       - name: Test with pytest
         run: |
           python -m pytest -x

.gitignore

@@ -84,9 +84,6 @@ target/
 profile_default/
 ipython_config.py
 
-# pyenv
-.python-version
-
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies
@@ -132,3 +129,6 @@ dmypy.json
 .pyre/
 
 tests/data/reports/*
+
+# PyCharm indexes
+.idea

.python-version

@@ -0,0 +1 @@
+3.13.0

Dockerfile

@@ -1,5 +1,5 @@
 # Use slim for a smaller image size and install only the required packages
-FROM python:3.8-slim-buster
+FROM python:3.13-slim
 
 # Use the following on M1; for odbc connection to mssql.
 # FROM --platform=linux/amd64 python:3.8-slim-buster

Pipfile

@@ -31,9 +31,11 @@ requests = "~=2.32"
 slackclient = "~=2.9"
 sqlalchemy = "~=2.0"
 pymongo = "~=4.8.0"
+setuptools = "*"
+numpy = "*"
 
 [requires]
-python_version = "3.8"
+python_version = "3.13"
 
 [pipenv]
 allow_prereleases = true

lighthouse/classes/event_properties/definitions/biosero/wells_from_destination.py

@@ -23,7 +23,7 @@ def _is_valid_destination_coordinate_not_duplicated(self, wells):
         coordinates = [well["destination_coordinate"] for well in wells]
         duplicates = set([coor for coor in coordinates if coordinates.count(coor) > 1])
         if len(duplicates) > 0:
-            raise RetrievalError(f"Some coordinates have clashing samples/controls: { duplicates }")
+            raise RetrievalError(f"Some coordinates have clashing samples/controls: {duplicates}")
 
     @property
     def errors(self) -> List[str]:

lighthouse/classes/event_properties/definitions/samples_from_destination.py

@@ -35,7 +35,7 @@ def _well_samples(self):
     def _is_valid_no_duplicate_uuids(self, uuids):
         duplicates = set([uuid for uuid in uuids if uuids.count(uuid) > 1])
         if len(duplicates) > 0:
-            raise RetrievalError(f"There is duplication in the lh sample uuids provided: { list(duplicates) }")
+            raise RetrievalError(f"There is duplication in the lh sample uuids provided: {list(duplicates)}")
 
     def _get_sample_with_uuid(self, samples, uuid):
         for sample in samples:

lighthouse/classes/event_properties/interfaces/event_property_abstract.py

@@ -138,27 +138,27 @@ def retrieval_scope(self):
         Returns:
             ContextManager - A context specifically created to handle a retrieval process
         """
-        logger.info(f"At { self._source_code_position_for_logging() } - Start retrieval")
+        logger.info(f"At {self._source_code_position_for_logging()} - Start retrieval")
 
         try:
             if not self.is_valid():
-                raise ValidationError(f"At { self._source_code_position_for_logging() } - Exception during validation")
+                raise ValidationError(f"At {self._source_code_position_for_logging()} - Exception during validation")
             yield
         except Exception as exc:
-            logger.error(f"At { self._source_code_position_for_logging() } - Exception during retrieval")
+            logger.error(f"At {self._source_code_position_for_logging()} - Exception during retrieval")
 
             self._is_valid = False
             msg = f"Exception during retrieval: {exc}"
             if msg not in self._errors:
                 self._errors.append(msg)
             raise exc
 
-        logger.info(f"At { self._source_code_position_for_logging() } - End retrieval")
+        logger.info(f"At {self._source_code_position_for_logging()} - End retrieval")
 
     def _source_code_position_for_logging(self):
         klass = type(self)
         frame = sys._getframe(3)
         function_name = frame.f_code.co_name
         line_no = frame.f_lineno
 
-        return f"{ klass.__qualname__ }::{ function_name } - line { line_no }"
+        return f"{klass.__qualname__}::{function_name} - line {line_no}"

lighthouse/classes/event_properties/validations.py

@@ -17,7 +17,7 @@ def validate_param_not_missing(self, param: str) -> None:
             bool - True/False indicating if this condition is met
         """
         with self.validation_scope():
-            self.process_validation(self.get_param_value(param) is not None, f"'{ param }' is missing")
+            self.process_validation(self.get_param_value(param) is not None, f"'{param}' is missing")
 
     def validate_param_not_empty(self, param: str) -> None:
         """
@@ -31,7 +31,7 @@ def validate_param_not_empty(self, param: str) -> None:
             bool - True/False indicating if this condition is met
         """
         with self.validation_scope():
-            self.process_validation(self.get_param_value(param) != "", f"'{ param }' should not be an empty string")
+            self.process_validation(self.get_param_value(param) != "", f"'{param}' should not be an empty string")
 
     def validate_param_no_whitespaces(self, param: str) -> None:
         """
@@ -47,7 +47,7 @@ def validate_param_no_whitespaces(self, param: str) -> None:
         with self.validation_scope():
             text_to_check = self.get_param_value(param)
             self.process_validation(
-                text_to_check is None or (" " not in text_to_check), f"'{ param }' should not contain any whitespaces"
+                text_to_check is None or (" " not in text_to_check), f"'{param}' should not contain any whitespaces"
             )
 
     def validate_param_is_integer(self, param: str) -> None:
@@ -65,4 +65,4 @@ def validate_param_is_integer(self, param: str) -> None:
             bool - True/False indicating if this condition is met
         """
         with self.validation_scope():
-            self.process_validation(is_integer(self.get_param_value(param)), f"'{ param }' should be an integer")
+            self.process_validation(is_integer(self.get_param_value(param)), f"'{param}' should be an integer")

lighthouse/classes/events/biosero/destination_completed.py

@@ -78,4 +78,4 @@ def process_event(self) -> None:
         response = message.send_to_ss()
 
         if not response.ok:
-            raise Exception(f"There was some problem when sending message to Sequencescape: { response.text }")
+            raise Exception(f"There was some problem when sending message to Sequencescape: {response.text}")

lighthouse/classes/events/biosero/destination_partial_completed.py

@@ -78,4 +78,4 @@ def process_event(self) -> None:
         response = message.send_to_ss()
 
         if not response.ok:
-            raise Exception(f"There was some problem when sending message to Sequencescape: { response.text }")
+            raise Exception(f"There was some problem when sending message to Sequencescape: {response.text}")

lighthouse/classes/events/biosero/error_recovered_destination_completed.py

@@ -78,4 +78,4 @@ def process_event(self) -> None:
         response = message.send_to_ss()
 
         if not response.ok:
-            raise Exception(f"There was some problem when sending message to Sequencescape: { response.text }")
+            raise Exception(f"There was some problem when sending message to Sequencescape: {response.text}")

lighthouse/classes/events/biosero/error_recovered_destination_partial_completed.py

@@ -78,4 +78,4 @@ def process_event(self) -> None:
         response = message.send_to_ss()
 
         if not response.ok:
-            raise Exception(f"There was some problem when sending message to Sequencescape: { response.text }")
+            raise Exception(f"There was some problem when sending message to Sequencescape: {response.text}")

lighthouse/classes/services/cherrytrack.py

@@ -14,7 +14,7 @@ class CherrytrackServiceMixin:
     def raise_error_from_response(self, response):
         json = response.json()
         if not (isinstance(json, dict)):
-            raise Exception(f"Response from Cherrytrack is not a valid JSON: { json }")
+            raise Exception(f"Response from Cherrytrack is not a valid JSON: {json}")
 
         if json and ("errors" in json):
             raise Exception(f"Response from Cherrytrack is not OK: {','.join(json['errors'])}")

lighthouse/classes/services/labwhere.py

@@ -29,7 +29,7 @@ def transfer_to_bin(self: Any) -> None:
             user_barcode=robot_barcode,
         )
         if not response.ok:
-            raise Exception(f"There was a problem attempting to change a location in LabWhere: { response.text }")
+            raise Exception(f"There was a problem attempting to change a location in LabWhere: {response.text}")
 
     @staticmethod
     def _destroyed_barcode() -> str:

lighthouse/config/defaults.py

@@ -184,7 +184,7 @@
 
 # NB: Remember to copy this definition to any config which redefines any of the variables that are used to create it.
 DART_SQL_SERVER_CONNECTION_STRING = (
-    "DRIVER={ODBC Driver 17 for SQL Server};"
+    "DRIVER={ODBC Driver 18 for SQL Server};"
     f"SERVER=tcp:{DART_SQL_SERVER_HOST};"
     f"DATABASE={DART_SQL_SERVER_DATABASE};"
     f"UID={DART_SQL_SERVER_USER};"

lighthouse/config/test.py

@@ -65,11 +65,12 @@
 # NB: Create the connection string here as we define the database here. Since a f-string is evaluated immediately,
 # it cannot only live in defaults.py if we redefine any of the variables that are used to create it.
 DART_SQL_SERVER_CONNECTION_STRING = (
-    "DRIVER={ODBC Driver 17 for SQL Server};"
+    "DRIVER={ODBC Driver 18 for SQL Server};"
     f"SERVER=tcp:{DART_SQL_SERVER_HOST};"
     f"DATABASE={DART_SQL_SERVER_DATABASE};"
     f"UID={DART_SQL_SERVER_USER};"
-    f"PWD={DART_SQL_SERVER_PASSWORD}"
+    f"PWD={DART_SQL_SERVER_PASSWORD};"
+    f"TrustServerCertificate=yes;"
 )
 
 ###

setup_sqlserver_test_db.py

@@ -6,9 +6,9 @@
 
 cnxn = pyodbc.connect(
     (
-        "DRIVER={ODBC Driver 17 for SQL Server};"  # noqa: F541
+        "DRIVER={ODBC Driver 18 for SQL Server};"  # noqa: F541
         f"SERVER=tcp:{LOCALHOST};"
-        "DATABASE=master;UID=SA;PWD=MyS3cr3tPassw0rd"
+        "DATABASE=master;UID=SA;PWD=MyS3cr3tPassw0rd;TrustServerCertificate=yes"
     ),
     autocommit=True,
 )

tests/endpoints/events/beckman/test_source_completed.py

@@ -77,7 +77,7 @@ def test_get_event_source_completed(
 
                     mocked_rabbit_channel.basic_publish.assert_called_with(
                         exchange="lighthouse.test.examples",
-                        routing_key=f"test.event.{ Beckman.EVENT_SOURCE_COMPLETED }",
+                        routing_key=f"test.event.{Beckman.EVENT_SOURCE_COMPLETED}",
                         body='{"event": {"uuid": "'
                         + int_to_uuid(1)
                         + (

tests/endpoints/events/beckman/test_source_no_pickable_wells.py

@@ -79,7 +79,7 @@ def test_get_event_source_no_pickable_wells(
 
                     mocked_rabbit_channel.basic_publish.assert_called_with(
                         exchange="lighthouse.test.examples",
-                        routing_key=f"test.event.{ Beckman.EVENT_SOURCE_ALL_NEGATIVES }",
+                        routing_key=f"test.event.{Beckman.EVENT_SOURCE_ALL_NEGATIVES}",
                         body='{"event": {"uuid": "'
                         + int_to_uuid(1)
                         + (

tests/endpoints/events/beckman/test_source_no_plate_map_data.py

@@ -54,7 +54,7 @@ def test_event_no_plate_map_data(
 
                     mocked_rabbit_channel.basic_publish.assert_called_with(
                         exchange="lighthouse.test.examples",
-                        routing_key=f"test.event.{ Beckman.EVENT_SOURCE_NO_PLATE_MAP_DATA }",
+                        routing_key=f"test.event.{Beckman.EVENT_SOURCE_NO_PLATE_MAP_DATA}",
                         body='{"event": {"uuid": "'
                         + int_to_uuid(1)
                         + (

tests/endpoints/events/beckman/test_source_unrecognised.py

@@ -38,7 +38,7 @@ def test_get_event_source_unrecognised(
 
                     mocked_rabbit_channel.basic_publish.assert_called_with(
                         exchange="lighthouse.test.examples",
-                        routing_key=f"test.event.{ Beckman.EVENT_SOURCE_UNRECOGNISED }",
+                        routing_key=f"test.event.{Beckman.EVENT_SOURCE_UNRECOGNISED}",
                         body='{"event": {"uuid": "'
                         + int_to_uuid(1)
                         + (

tests/endpoints/events/biosero/test_destination_completed.py

@@ -160,7 +160,7 @@ def test_post_event_destination_completed(
 
                         mocked_rabbit_channel.basic_publish.assert_called_with(
                             exchange="lighthouse.test.examples",
-                            routing_key=f"test.event.{ Biosero.EVENT_DESTINATION_COMPLETED }",
+                            routing_key=f"test.event.{Biosero.EVENT_DESTINATION_COMPLETED}",
                             body=event_message,
                         )
 

tests/endpoints/events/biosero/test_destination_failed.py

@@ -112,7 +112,7 @@ def test_post_event_partially_completed(
 
                     mocked_rabbit_channel.basic_publish.assert_called_with(
                         exchange="lighthouse.test.examples",
-                        routing_key=f"test.event.{ Biosero.EVENT_DESTINATION_FAILED }",
+                        routing_key=f"test.event.{Biosero.EVENT_DESTINATION_FAILED}",
                         body=event_message,
                     )
 

tests/endpoints/events/biosero/test_destination_partial.py

@@ -31,7 +31,7 @@ def test_post_destination_completed_missing_barcode(app, client, biosero_auth_he
         assert response.json == {
             "_status": "ERR",
             "_issues": {
-                "event_type": (f"'barcode' cannot be empty with the '{ Biosero.EVENT_DESTINATION_PARTIAL }' event")
+                "event_type": (f"'barcode' cannot be empty with the '{Biosero.EVENT_DESTINATION_PARTIAL}' event")
             },
             "_error": {"code": 422, "message": "Insertion failure: 1 document(s) contain(s) error(s)"},
         }
@@ -51,7 +51,7 @@ def test_post_destination_partial_missing_run_id(app, client, biosero_auth_heade
         assert response.json == {
             "_status": "ERR",
             "_issues": {
-                "event_type": f"'{ Biosero.EVENT_DESTINATION_PARTIAL }' requires a corresponding 'run_id' parameter"
+                "event_type": f"'{Biosero.EVENT_DESTINATION_PARTIAL}' requires a corresponding 'run_id' parameter"
             },
             "_error": {"code": 422, "message": "Insertion failure: 1 document(s) contain(s) error(s)"},
         }
@@ -133,7 +133,7 @@ def test_post_event_partial(
 
                     mocked_rabbit_channel.basic_publish.assert_called_with(
                         exchange="lighthouse.test.examples",
-                        routing_key=f"test.event.{ Biosero.EVENT_DESTINATION_PARTIAL }",
+                        routing_key=f"test.event.{Biosero.EVENT_DESTINATION_PARTIAL}",
                         body=event_message,
                     )
 

tests/endpoints/events/biosero/test_destination_partial_completed.py

@@ -147,7 +147,7 @@ def test_post_event_destination_partially_completed(
 
                         mocked_rabbit_channel.basic_publish.assert_called_with(
                             exchange="lighthouse.test.examples",
-                            routing_key=f"test.event.{ Biosero.EVENT_DESTINATION_PARTIAL_COMPLETED }",
+                            routing_key=f"test.event.{Biosero.EVENT_DESTINATION_PARTIAL_COMPLETED}",
                             body=event_message,
                         )