fix the pytest

app/__init__.py

@@ -0,0 +1,22 @@
+from flask import Flask
+from .models import db
+from flask_migrate import Migrate
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DATABASE_URI = os.getenv('DATABASE_URI')
+
+def create_app():
+    app = Flask(__name__)
+    app.config['SQLALCHEMY_DATABASE_URI'] = DATABASE_URI
+    db.init_app(app)
+
+    # Initialize Flask-Migrate
+    Migrate(app, db)
+
+    from .routes import register_routes
+    register_routes(app)
+
+    return app

app/flask-app-structure.txt

@@ -0,0 +1,22 @@
+DATAGOV-HARVESTING-LOGIC
+├── app/
+│   ├── __init__.py
+│   ├── models.py
+│   ├── routes.py
+│   ├── forms.py (to-do)
+│   └── templates/
+│       ├── index.html
+│       ├── harvest_source_form.html (to-do)
+│       └── xxx.html (to-do)
+│   └── static/
+│       └── styles.css (to-do)
+│
+├── migrations/
+│   └── versions/
+│   ├── alembic.ini
+│   ├── env.py
+│   └── script.py.mako
+│
+├── docker-compose.yml
+├── Dockerfile
+└── run.py

app/interface.py

@@ -0,0 +1,93 @@
+from sqlalchemy import create_engine, inspect
+from sqlalchemy.orm import sessionmaker, scoped_session
+from app.models import Organization, HarvestSource, HarvestJob, HarvestError
+from . import DATABASE_URI
+
+class HarvesterDBInterface:
+    def __init__(self, session=None):
+        if session is None:
+            engine = create_engine(DATABASE_URI)
+            session_factory = sessionmaker(bind=engine,
+                                           autocommit=False,
+                                           autoflush=False)
+            self.db = scoped_session(session_factory)
+        else:
+            self.db = session
+
+    @staticmethod
+    def _to_dict(obj):
+        return {c.key: getattr(obj, c.key) 
+                for c in inspect(obj).mapper.column_attrs}
+
+    def add_organization(self, org_data):
+        new_org = Organization(**org_data)
+        self.db.add(new_org)
+        self.db.commit()
+        self.db.refresh(new_org)
+        return new_org
+
+    def add_harvest_source(self, source_data, org_id):
+        source_data['organization_id'] = org_id
+        new_source = HarvestSource(**source_data)
+        self.db.add(new_source)
+        self.db.commit()
+        self.db.refresh(new_source)
+        return new_source
+
+    def get_all_organizations(self):
+        orgs = self.db.query(Organization).all()
+        orgs_data = [
+            HarvesterDBInterface._to_dict(org) for org in orgs]
+        return orgs_data
+
+    def get_all_harvest_sources(self):
+        harvest_sources = self.db.query(HarvestSource).all()
+        harvest_sources_data = [
+            HarvesterDBInterface._to_dict(source) for source in harvest_sources]
+        return harvest_sources_data
+
+    def get_harvest_source(self, source_id):
+        result = self.db.query(HarvestSource).filter_by(id=source_id).first()
+        return HarvesterDBInterface._to_dict(result)
+
+    def add_harvest_job(self, job_data, source_id):
+        job_data['harvest_source_id'] = source_id
+        new_job = HarvestJob(**job_data)
+        self.db.add(new_job)
+        self.db.commit()
+        self.db.refresh(new_job)
+        return new_job
+
+    def get_all_harvest_jobs(self):
+        harvest_jobs = self.db.query(HarvestJob).all()
+        harvest_jobs_data = [
+            HarvesterDBInterface._to_dict(job) for job in harvest_jobs]
+        return harvest_jobs_data
+
+    def get_harvest_job(self, job_id):
+        result = self.db.query(HarvestJob).filter_by(id=job_id).first()
+        return HarvesterDBInterface._to_dict(result)
+
+    def add_harvest_error(self, error_data, job_id):
+        error_data['harvest_job_id'] = job_id
+        new_error = HarvestError(**error_data)
+        self.db.add(new_error)
+        self.db.commit()
+        self.db.refresh(new_error)
+        return new_error
+
+    def get_all_harvest_errors_by_job(self, job_id):
+        harvest_errors = self.db.query(HarvestError).filter_by(harvest_job_id=job_id)
+        harvest_errors_data = [
+            HarvesterDBInterface._to_dict(err) for err in harvest_errors]
+        return harvest_errors_data
+
+    def get_harvest_error(self, error_id):
+        result = self.db.query(HarvestError).filter_by(id=error_id).first()
+        return HarvesterDBInterface._to_dict(result)
+
+    def close(self):
+        if hasattr(self.db, 'remove'):
+            self.db.remove()
+        elif hasattr(self.db, 'close'):
+            self.db.close()

app/models.py

@@ -0,0 +1,82 @@
+from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy.dialects.postgresql import UUID, ARRAY
+from sqlalchemy.sql import text
+from sqlalchemy import Enum
+from sqlalchemy.schema import Index
+
+db = SQLAlchemy()
+
+class Base(db.Model):
+    __abstract__ = True  # Indicates that this class should not be created as a table
+    id = db.Column(UUID(as_uuid=True), primary_key=True,
+                   server_default=text("gen_random_uuid()"))
+
+class Organization(Base):
+    __tablename__ = 'organization'
+
+    name = db.Column(db.String(), nullable=False, index=True)
+    logo = db.Column(db.String())
+
+class HarvestSource(Base):
+    __tablename__ = 'harvest_source'
+
+    name = db.Column(db.String, nullable=False)
+    notification_emails = db.Column(ARRAY(db.String))
+    organization_id = db.Column(UUID(as_uuid=True),
+                                db.ForeignKey('organization.id'),
+                                nullable=False)
+    frequency = db.Column(db.String, nullable=False)
+    url = db.Column(db.String, nullable=False, unique=True)
+    schema_type = db.Column(db.String, nullable=False)
+    source_type = db.Column(db.String, nullable=False)
+    jobs = db.relationship('HarvestJob', backref='source')
+
+class HarvestJob(Base):
+    __tablename__ = 'harvest_job'
+
+    harvest_source_id = db.Column(UUID(as_uuid=True),
+                                  db.ForeignKey('harvest_source.id'),
+                                  nullable=False)
+    status = db.Column(Enum('new', 'in_progress', 'complete', name='job_status'),
+                       nullable=False,
+                       index=True)
+    date_created = db.Column(db.DateTime, index=True)
+    date_finished = db.Column(db.DateTime)
+    records_added = db.Column(db.Integer)
+    records_updated = db.Column(db.Integer)
+    records_deleted = db.Column(db.Integer)
+    records_errored = db.Column(db.Integer)
+    records_ignored = db.Column(db.Integer)
+    errors = db.relationship('HarvestError', backref='job', lazy=True)
+
+class HarvestError(Base):
+    __tablename__ = 'harvest_error'
+
+    harvest_job_id = db.Column(UUID(as_uuid=True),
+                               db.ForeignKey('harvest_job.id'),
+                               nullable=False)
+    harvest_record_id = db.Column(db.String)
+    # to-do 
+    # harvest_record_id = db.Column(UUID(as_uuid=True),
+    #                               db.ForeignKey('harvest_record.id'),
+    #                               nullable=True)
+    date_created = db.Column(db.DateTime)
+    type = db.Column(db.String)
+    severity = db.Column(Enum('CRITICAL', 'ERROR', 'WARN', name='error_serverity'),
+                         nullable=False,
+                         index=True)
+    message = db.Column(db.String)
+
+class HarvestRecord(Base):
+    __tablename__ = 'harvest_record'
+
+    job_id = db.Column(UUID(as_uuid=True),
+                       db.ForeignKey('harvest_job.id'),
+                       nullable=False) 
+    identifier = db.Column(db.String(), nullable=False)
+    ckan_id = db.Column(db.String(), nullable=False, index=True)
+    type = db.Column(db.String(), nullable=False)
+    source_metadata = db.Column(db.String(), nullable=True)
+    __table_args__ = (
+        Index('ix_job_id_identifier', 'job_id', 'identifier'),
+    )

app/routes.py

@@ -0,0 +1,92 @@
+from flask import Blueprint, request, render_template
+from .interface import HarvesterDBInterface
+from tests.database.data import new_org, new_source, new_job, new_error
+
+mod = Blueprint('harvest', __name__)
+db = HarvesterDBInterface()
+
+@mod.route('/', methods=['GET'])
+def index():
+    return render_template('index.html')
+
+@mod.route('/add_org', methods=['POST', 'GET'])
+def add_organization():
+    org=db.add_organization(new_org)
+    return(f"Added new organization with ID: {org.id}")
+
+@mod.route('/add_source', methods=['POST', 'GET'])
+def add_harvest_source():
+    org_id = request.args.get('org_id', None)
+    if org_id is None:
+        return 'Please provide org_id: /add_source?org_id=xxx'
+    else:
+        source=db.add_harvest_source(new_source, org_id)
+        return(f"Added new source with ID: {source.id}")
+
+@mod.route('/add_job', methods=['POST', 'GET'])
+def add_harvest_job():
+    source_id = request.args.get('source_id', None)
+    if source_id is None:
+        return 'Please provide source_id: /add_job?source_id=xxx'
+    else:
+        job=db.add_harvest_job(new_job, source_id)
+        return(f"Added new job with ID: {job.id}")
+
+@mod.route('/add_error', methods=['POST', 'GET'])
+def add_harvest_error():
+    job_id = request.args.get('job_id', None)
+    if job_id is None:
+        return 'Please provide job_id: /add_error?job_id=xxx'
+    else:
+        err=db.add_harvest_error(new_error, job_id)
+        return(f"Added new error with ID: {err.id}")
+
+@mod.route('/organizations', methods=['GET'])
+def get_all_organizations():
+    result = db.get_all_organizations()
+    return result
+
+@mod.route('/harvest_sources', methods=['GET'])
+def get_all_harvest_sources():
+    result = db.get_all_harvest_sources()
+    return result
+
+@mod.route('/harvest_jobs', methods=['GET'])
+def get_all_harvest_jobs():
+    result = db.get_all_harvest_jobs()
+    return result
+
+@mod.route('/harvest_errors_by_job/<job_id>', methods=['GET'])
+def get_all_harvest_errors_by_job(job_id):
+    try:
+        result = db.get_all_harvest_errors_by_job(job_id)
+        return result
+    except Exception:
+        return " provide job_id"
+
+@mod.route('/harvest_source/<source_id>', methods=['GET'])
+def get_harvest_source(source_id):
+    try:
+        result = db.get_harvest_source(source_id)
+        return result
+    except Exception:
+        return " provide source_id"
+
+@mod.route('/harvest_job/<job_id>', methods=['GET'])
+def get_harvest_job(job_id):
+    try:
+        result = db.get_harvest_job(job_id)
+        return result
+    except Exception:
+        return "provide job_id"
+
+@mod.route('/harvest_error/<error_id>', methods=['GET'])
+def get_harvest_error(error_id):
+    try:
+        result = db.get_harvest_error(error_id)
+        return result
+    except Exception:
+        return "provide error_id"
+
+def register_routes(app):
+    app.register_blueprint(mod)

app/templates/index.html

@@ -0,0 +1,38 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Harvest Actions</title>
+        <style>
+            ul {
+                list-style-type: none;
+            }
+            li {
+                margin-bottom: 10px;
+            }
+            a {
+                text-decoration: none;
+                color: blue;
+            }
+            a:hover {
+                text-decoration: underline;
+            }
+        </style>
+</head>
+<body>
+    <h2>Harvest Actions</h2>
+    <ul>
+        <li><a href="/add_org">Add Organization</a></li>
+        <li><a href="/add_source">Add Harvest Source</a></li>
+        <li><a href="/add_job">Add Harvest Job</a></li>
+        <li><a href="/add_error">Add Harvest Error</a></li>
+        <li><a href="/organizations">Get All Organizations</a></li>
+        <li><a href="/harvest_sources">Get All Harvest Sources</a></li>
+        <li><a href="/harvest_jobs">Get All Harvest Jobs</a></li>
+        <li><a href="/harvest_errors_by_job/<job_id>">Get All Harvest Errors By Job</a></li>
+        <li><a href="/harvest_source/<source_id>">Get Harvest Source</a></li>
+        <li><a href="/harvest_job/<job_id>">Get Harvest Job</a></li>
+        <li><a href="/harvest_error/<error_id>">Get Harvest Error</a></li>
+    </ul>
+</body>
+</html>

migrations/README

@@ -0,0 +1 @@
+Single-database configuration for Flask.