initial repo refactor

.github/workflows/builder.yaml

@@ -3,7 +3,8 @@ name: Builder
 # TODO: figure out a more wildcard-y way to do this
 env:
   BUILD_ARGS: "--test"
-  MONITORED_FILES: "build.yaml config.yaml Dockerfile run_tracking.sh run_gateway.sh run_multiserver.sh requirements.txt config.py route-config.yaml main.py"
+  MINDCTRL_SRC_DIR: "python/src/"
+  MINDCTRL_SERVICES_DIR: "services/"
 
 on:
   push:
@@ -34,7 +35,7 @@ jobs:
         run: |
           declare -a changed_addons
           for addon in ${{ steps.addons.outputs.addons }}; do
-            if [[ "${{ steps.changed_files.outputs.all }}" =~ $addon/ ]]; then
+            if [[ "${{ steps.changed_files.outputs.all }}" =~ $addon/ ]] || [[ "${{ steps.changed_files.outputs.all }}" =~ "${MINDCTRL_SRC_DIR}" ]] || [[ "${{ steps.changed_files.outputs.all }}" =~ "${MINDCTRL_SERVICES_DIR}" ]]; then
               if [[ ! "${changed_addons[@]}" =~ $addon ]]; then
                 changed_addons+=("\"${addon}\",");
               fi
@@ -48,7 +49,7 @@ jobs:
             echo "changed=true" >> $GITHUB_OUTPUT;
             echo "addons=[$changed]" >> $GITHUB_OUTPUT;
           else
-            echo "No add-on had any monitored files changed (${{ env.MONITORED_FILES }})";
+            echo "No add-on had any monitored files changed (${{ steps.changed_files.outputs.all }})";
           fi
   build:
     needs: init
@@ -88,6 +89,10 @@ jobs:
              echo "build_arch=false" >> $GITHUB_OUTPUT;
           fi
 
+      - name: Copy python source + services into build context because builder action doesn't support --build-context
+        id: copy
+        run: bash scripts/copy_content_to_addon_context.sh
+
       - name: Login to GitHub Container Registry
         if: env.BUILD_ARGS != '--test'
         uses: docker/login-action@v3.1.0

.github/workflows/integration-k3d.yaml

@@ -45,5 +45,5 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
-          pytest -s -v -c pytest.ini --deploy-mode=k3d
+          pytest -s -v -c pytest.ini --deploy-mode=k3d --replay-mode=replay
         working-directory: mindctrl-addon/tests

.github/workflows/integration.yaml

@@ -33,4 +33,4 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
-          pytest -v -s -c mindctrl-addon/tests/pytest.ini
+          pytest -v -s -c mindctrl-addon/tests/pytest.ini --deploy-mode=local --replay-mode=replay

.vscode/settings.json

@@ -4,5 +4,8 @@
         "mindctrl-addon/tests/pytest.ini"
     ],
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
+    "python.testing.pytestEnabled": true,
+    "python.analysis.extraPaths": [
+        "python/src"
+    ]
 }

mindctrl-addon/Dockerfile

@@ -48,6 +48,10 @@ RUN mkdir /dapr; \
 RUN dapr init --slim
 
 COPY rootfs /
+# One day when builder workflow supports --build-context
+# COPY --from=mindctrl_source / /pysrc
+COPY pysrc /
+RUN pip3 install --no-cache --break-system-packages /pysrc
 
 RUN chmod a+x /usr/bin/run_tracking.sh && \
     chmod a+x /usr/bin/run_gateway.sh && \

mindctrl-addon/requirements.txt

@@ -1,22 +1,3 @@
-# https://raw.githubusercontent.com/mlflow/mlflow/master/requirements/gateway-requirements.txt
-
-mlflow[genai]~=2.11
-pydantic<3,>=1.0
-pydantic-settings~=2.2
-fastapi<1
-uvicorn[standard]<1
-watchfiles<1
-aiohttp<4
-boto3<2,>=1.28.56
-openai<1
-tiktoken
-tenacity
-# homeassistant>2023.11 # TODO: I need to switch to Arch from debian :(
-# https://aur.archlinux.org/packages/python-mlflow
-azureml-mlflow
-aiomqtt
-pgvector
-asyncpg
-# transformers
-# torch
-# sentence-transformers
+-r /services/deployments/requirements.txt
+-r /services/multiserver/requirements.txt
+-r /services/tracking/requirements.txt

mindctrl-addon/rootfs/usr/bin/run_gateway.sh

@@ -11,4 +11,4 @@ bashio::log.info "Starting MLflow Deployment Server with Dapr..."
 # https://github.com/dapr/dashboard/issues/195
 s6-notifyoncheck dapr run --app-id deployments --app-port 5001 --app-protocol http \
   --enable-api-logging --enable-app-health-check --app-health-check-path /health --dapr-http-port 5501 -- \
-  mlflow deployments start-server --config-path /usr/bin/deployments/route-config.yaml --port 5001 --host 0.0.0.0
+  mlflow deployments start-server --config-path /services/deployments/route-config.yaml --port 5001 --host 0.0.0.0

mindctrl-addon/rootfs/usr/bin/run_multiserver.sh

@@ -38,7 +38,8 @@ bashio::log.info "setting notification fd to ${notifyfd}"
 export NOTIFY_FD="${notifyfd}"
 
 export MLFLOW_TRACKING_URI="http://0.0.0.0:5000"
-export PYTHONPATH="/usr/bin/multiserver"
+export MLFLOW_DEPLOYMENTS_TARGET="http://0.0.0.0:5001"
+# export PYTHONPATH="/usr/bin/multiserver"
 
 bashio::log.info "Starting MLflow Tracking Server with Dapr..."
-dapr run --app-id multiserver --app-port 5002 -- python3 -m uvicorn main:app --host 0.0.0.0 --port 5002
+dapr run --app-id multiserver --app-port 5002 -- python3 -m uvicorn mindctrl.main:app --host 0.0.0.0 --port 5002

mindctrl-addon/test-requirements.txt

@@ -11,4 +11,6 @@ ruff
 # already pulled in by testcontainers, but we need it separately too
 # ALSO: https://github.com/docker/docker-py/issues/3240
 docker<7
+# ugh..https://github.com/docker/docker-py/issues/2230
+python-on-whales
 -r requirements.txt