Fix e2e tests by distinguishing kedro-datasets dependency for diffe…

…rent python versions (#3802)


Signed-off-by: Merel Theisen <merel.theisen@quantumblack.com>

Makefile

@@ -46,7 +46,7 @@ package: clean install
 
 install-test-requirements:
 	python -m pip install -U "pip>=21.2"
-	pip install .[test]
+	pip install -U .[test]
 
 install-pre-commit:
 	pre-commit install --install-hooks

features/environment.py

@@ -5,6 +5,7 @@
 import os
 import shutil
 import subprocess
+import sys
 import tempfile
 import venv
 from pathlib import Path
@@ -14,6 +15,7 @@
 _PATHS_TO_REMOVE: set[Path] = set()
 
 FRESH_VENV_TAG = "fresh_venv"
+MINOR_PYTHON_38_VERSION = 8
 
 
 def call(cmd, env):
@@ -130,6 +132,11 @@ def _install_project_requirements(context):
         .splitlines()
     )
     install_reqs = [req for req in install_reqs if "{" not in req and "#" not in req]
-    install_reqs.append("kedro-datasets[pandas-csvdataset]")
+    # For Python versions 3.9 and above we use the new dataset dependency format introduced in `kedro-datasets` 3.0.0
+    if sys.version_info.minor > MINOR_PYTHON_38_VERSION:
+        install_reqs.append("kedro-datasets[pandas-csvdataset]")
+    # For Python 3.8 we use the older `kedro-datasets` dependency format
+    else:
+        install_reqs.append("kedro-datasets[pandas.CSVDataset]")
     call([context.pip, "install", *install_reqs], env=context.env)
     return context

features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt

@@ -3,7 +3,8 @@ ipython>=8.10
 jupyterlab>=3.0
 notebook
 kedro~={{ cookiecutter.kedro_version}}
-kedro-datasets[pandas-csvdataset]
+kedro-datasets[pandas-csvdataset]; python_version >= "3.9"
+kedro-datasets[pandas.CSVDataset]<2.0.0; python_version < '3.9'
 kedro-telemetry>=0.3.1
 pytest-cov~=3.0
 pytest-mock>=1.7.1, <2.0

pyproject.toml

@@ -61,7 +61,8 @@ test = [
     "jupyterlab_server>=2.11.1",
     "jupyterlab>=3,<5",
     "jupyter~=1.0",
-    "kedro-datasets",
+    "kedro-datasets; python_version >= '3.9'",
+    "kedro-datasets<2.0.0; python_version < '3.9'",
     "mypy~=1.0",
     "pandas~=2.0",
     "pluggy>=1.0, <1.4", # pluggy 1.4 hide imports inside function and causing mocking issue