add some indexes

GSA · Mar 19, 2024 · 719c58c · 719c58c
1 parent 8d5bd69
commit 719c58c
Show file tree

Hide file tree

Showing 20 changed files with 253 additions and 353 deletions.
diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml
@@ -74,15 +74,15 @@ jobs:
       - name: deploy DHL
         uses: cloud-gov/cg-cli-tools@main
         with:
-          command: cf push datagov-harvesting-logic --vars-file vars.development.yml --strategy rolling --no-wait
+          command: cf push --vars-file vars.development.yml --strategy rolling --no-wait
           cf_org: gsa-datagov
           cf_space: ${{vars.ENVIRONMENT_NAME}}
           cf_username: ${{secrets.CF_SERVICE_USER}}
           cf_password: ${{secrets.CF_SERVICE_AUTH}}
       - name: smoke test
         uses: cloud-gov/cg-cli-tools@main
         with:
-          command: cf run-task datagov-harvesting-logic -c "/home/vcap/app/scripts/smoke-test.py" --name smoke-test
+          command: cf run-task harvesting-logic -c "/home/vcap/app/scripts/smoke-test.py" --name smoke-test
           cf_org: gsa-datagov
           cf_space: ${{vars.ENVIRONMENT_NAME}}
           cf_username: ${{secrets.CF_SERVICE_USER}}
@@ -91,7 +91,7 @@ jobs:
         uses: cloud-gov/cg-cli-tools@main
         with:
           command: >
-            scripts/monitor-cf-logs.sh datagov-harvesting-logic smoke-test
+            scripts/monitor-cf-logs.sh harvesting-logic smoke-test
           cf_org: gsa-datagov
           cf_space: ${{vars.ENVIRONMENT_NAME}}
           cf_username: ${{secrets.CF_SERVICE_USER}}

diff --git a/.gitignore b/.gitignore
@@ -24,4 +24,5 @@ tmp/
 # vscode debugger
 .vscode/
 .env
+requirements.txt
 
diff --git a/Dockerfile b/Dockerfile
@@ -8,4 +8,7 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 EXPOSE 8080
 
-CMD ["python", "app.py"]
+ENV FLASK_APP=run.py
+
+# Run run.py when the container launches
+CMD ["flask", "run", "--host=0.0.0.0", "--port=8080"]
diff --git a/README.md b/README.md
@@ -107,6 +107,14 @@ If you followed the instructions for `CKAN load testing` and `Harvester testing`
 
    This will start the necessary services and execute the test.
 
+3. when there are database DDL changes, use following steps to generate migration scripts and update database:
+
+    ```bash
+    docker-compose db up
+    docker-compose run app flask db migrate -m "migration description"
+    docker-compose run app flask db upgrade
+    ```
+
 ### Deployment to cloud.gov
 
 #### Database Service Setup
@@ -143,4 +151,10 @@ Accessing the service can be done with service keys. They can be created with `c
    ```bash
    poetry export -f requirements.txt --output requirements.txt --without-hashes
    cf push --vars-file vars.development.yml
-   ```
+   ```
+
+3. when there are database DDL changes, use following to do the database update:
+
+    ```bash
+    cf run-task harvesting-logic --command "flask db upgrade" --name database-upgrade
+    ```
diff --git a/app.py b/app.py
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -54,10 +54,14 @@ services:
     build: .
     depends_on:
       - db
+    volumes:
+      - .:/app
     environment:
       DATABASE_URI: ${DATABASE_URI}
+      FLASK_APP: run.py
     ports:
       - "8080:8080"
+    command: flask run --host=0.0.0.0 --port=8080
 
 volumes:
   postgres_data:
diff --git a/harvester/database/__init__.py b/harvester/database/__init__.py
diff --git a/harvester/database/init_db.py b/harvester/database/init_db.py
diff --git a/harvester/database/interface.py b/harvester/database/interface.py
diff --git a/harvester/database/models.py b/harvester/database/models.py
diff --git a/harvester/exceptions.py b/harvester/exceptions.py
@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime
-from .database.interface import HarvesterDBInterface
+from app.interface import HarvesterDBInterface
 
 
 # critical exceptions
@@ -60,7 +60,7 @@ def __init__(self, msg, harvest_job_id, title):
             "severity": self.severity,
             "type": self.type,
             "date_created": datetime.utcnow(),
-            "record_id": self.title,
+            "harvest_record_id": self.title # to-do 
         }
 
         self.db_interface.add_harvest_error(error_data, self.harvest_job_id)

diff --git a/manifest.yml b/manifest.yml
@@ -10,4 +10,5 @@ applications:
     instances: 1
     env:
       DATABASE_NAME: ((database_name))
-    command: python app.py
+      FLASK_APP: run.py
+    command: flask run --host=0.0.0.0 --port=8080