merge from harvest-db-update

.env

@@ -8,7 +8,7 @@ S3FILESTORE__SIGNATURE_VERSION=s3v4
 
 MDTRANSLATOR_URL=http://127.0.0.1:3000/translates
 
-DATABASE_SERVER=db
+DATABASE_SERVER=localhost
 DATABASE_PORT=5432
 DATABASE_NAME=mydb
 DATABASE_USER=myuser

app/__init__.py

@@ -2,6 +2,9 @@
 from .models import db
 from flask_migrate import Migrate
 import os
+from dotenv import load_dotenv
+
+load_dotenv()
 
 DATABASE_URI = os.getenv('DATABASE_URI')
 

app/models.py

@@ -55,9 +55,11 @@ class HarvestError(Base):
     harvest_job_id = db.Column(UUID(as_uuid=True),
                                db.ForeignKey('harvest_job.id'),
                                nullable=False)
-    harvest_record_id = db.Column(UUID(as_uuid=True),
-                                  db.ForeignKey('harvest_record.id'),
-                                  nullable=True)
+    harvest_record_id = db.Column(db.String)
+    # to-do 
+    # harvest_record_id = db.Column(UUID(as_uuid=True),
+    #                               db.ForeignKey('harvest_record.id'),
+    #                               nullable=True)
     date_created = db.Column(db.DateTime)
     type = db.Column(db.String)
     severity = db.Column(Enum('CRITICAL', 'ERROR', 'WARN', name='error_serverity'),

app/templates/index.html

@@ -25,11 +25,14 @@ <h2>Harvest Actions</h2>
         <li><a href="/add_org">Add Organization</a></li>
         <li><a href="/add_source">Add Harvest Source</a></li>
         <li><a href="/add_job">Add Harvest Job</a></li>
+        <li><a href="/add_error">Add Harvest Error</a></li>
         <li><a href="/organizations">Get All Organizations</a></li>
         <li><a href="/harvest_sources">Get All Harvest Sources</a></li>
         <li><a href="/harvest_jobs">Get All Harvest Jobs</a></li>
+        <li><a href="/harvest_errors_by_job/<job_id>">Get All Harvest Errors By Job</a></li>
         <li><a href="/harvest_source/<source_id>">Get Harvest Source</a></li>
         <li><a href="/harvest_job/<job_id>">Get Harvest Job</a></li>
+        <li><a href="/harvest_error/<error_id>">Get Harvest Error</a></li>
     </ul>
 </body>
 </html>

harvester/__init__.py

@@ -1 +1,7 @@
-from .harvest import HarvestSource, Record # noqa
+import logging.config
+from harvester.logger_config import LOGGING_CONFIG
+from dotenv import load_dotenv
+
+load_dotenv()
+
+logging.config.dictConfig(LOGGING_CONFIG)

harvester/exceptions.py

@@ -0,0 +1,83 @@
+import logging
+from datetime import datetime
+from app.interface import HarvesterDBInterface
+
+
+# critical exceptions
+class HarvestCriticalException(Exception):
+    def __init__(self, msg, harvest_job_id):
+        super().__init__(msg, harvest_job_id)
+
+        self.msg = msg
+        self.harvest_job_id = harvest_job_id
+        self.severity = "CRITICAL"
+        self.type = "job"
+
+        self.db_interface = HarvesterDBInterface()
+        self.logger = logging.getLogger("harvest_runner")
+
+        error_data = {
+            "harvest_job_id": self.harvest_job_id,
+            "message": self.msg,
+            "severity": self.severity,
+            "type": self.type,
+            "date_created": datetime.utcnow(),
+        }
+
+        self.db_interface.add_harvest_error(error_data, self.harvest_job_id)
+        self.logger.critical(self.msg, exc_info=True)
+
+
+class ExtractHarvestSourceException(HarvestCriticalException):
+    pass
+
+
+class ExtractCKANSourceException(HarvestCriticalException):
+    pass
+
+
+class CompareException(HarvestCriticalException):
+    pass
+
+
+# non-critical exceptions
+class HarvestNonCriticalException(Exception):
+    def __init__(self, msg, harvest_job_id, title):
+        super().__init__(msg, harvest_job_id, title)
+
+        self.title = title
+        self.msg = msg
+        self.harvest_job_id = harvest_job_id
+        self.severity = "ERROR"
+        self.type = "record"
+
+        self.db_interface = HarvesterDBInterface()
+        self.logger = logging.getLogger("harvest_runner")
+
+        error_data = {
+            "harvest_job_id": self.harvest_job_id,
+            "message": self.msg,
+            "severity": self.severity,
+            "type": self.type,
+            "date_created": datetime.utcnow(),
+            "harvest_record_id": self.title # to-do 
+        }
+
+        self.db_interface.add_harvest_error(error_data, self.harvest_job_id)
+        self.logger.error(self.msg, exc_info=True)
+
+
+class ValidationException(HarvestNonCriticalException):
+    pass
+
+
+class TranformationException(HarvestNonCriticalException):
+    pass
+
+
+class DCATUSToCKANException(HarvestNonCriticalException):
+    pass
+
+
+class SynchronizeException(HarvestNonCriticalException):
+    pass