Update to Python3 (#154)

* chore(py3): run 2to3 and fix sorting

* use uc-cdis psqlgraph

* bump travis py3

* chore(py3): fix dependencies

* chore(py3): bump versions

* chore(py3): update dockerfile

* chore(py3): remove unused imports

* fix dependencies

* fix dockerfile

* edit uswgi file to use nginx

* dictionaryutils

* update uwsgi.ini

* python path

* update dockerfile

* ran black

* change docker cmd

* fix chown

* Remove editable vcs

* use pypi dependencies

* fix cdispyutils

* feedback py3

* TestDockerfile builds py3

* fix tests

* ran black

.github/workflows/wool.yml

@@ -0,0 +1,17 @@
+on:
+  pull_request:
+  issue_comment:
+    types: [created, edited]
+
+name: Wool
+
+jobs:
+  runWool:
+    name: Run black
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: uc-cdis/wool@master
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.travis.yml

@@ -3,7 +3,7 @@ sudo: false
 language: python
 
 python:
-  - "2.7"
+  - "3.6"
 
 cache:
   - pip

Dockerfile

@@ -1,24 +1,39 @@
 # To run: docker run -v /path/to/wsgi.py:/var/www/peregrine/wsgi.py --name=peregrine -p 81:80 peregrine
-# To check running container: docker exec -it peregrine /bin/bash 
+# To check running container: docker exec -it peregrine /bin/bash
 
-FROM quay.io/cdis/py27base:pybase2-1.0.2
+FROM quay.io/cdis/python-nginx:pybase3-1.1.0
 
-ENV DEBIAN_FRONTEND=noninteractive
+ENV appname=peregrine
 
-RUN mkdir /var/www/peregrine \
-    && chown www-data /var/www/peregrine
+RUN apk update \
+    && apk add postgresql-libs postgresql-dev libffi-dev libressl-dev \
+    && apk add linux-headers musl-dev gcc libxml2-dev libxslt-dev \
+    && apk add curl bash git vim
 
-COPY . /peregrine
+COPY . /$appname
 COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
-WORKDIR /peregrine
+WORKDIR /$appname
 
-RUN pip install -r requirements.txt \
-    && COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >peregrine/version_data.py \
-    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>peregrine/version_data.py
+RUN python -m pip install --upgrade pip \
+    && python -m pip install --upgrade setuptools \
+    && pip --version \
+    && pip install -r requirements.txt
+
+RUN mkdir -p /var/www/$appname \
+    && mkdir -p /var/www/.cache/Python-Eggs/ \
+    && mkdir /run/nginx/ \
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log \
+    && chown nginx -R /var/www/.cache/Python-Eggs/ \
+    && chown nginx /var/www/$appname
 
 EXPOSE 80
 
-WORKDIR /var/www/peregrine 
+RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py \
+    && python setup.py install
+
+
+WORKDIR /var/www/$appname
 
-ENTRYPOINT [ "/bin/sh", "/dockerrun.sh" ]
-CMD []
+CMD /dockerrun.sh

TestDockerfile

@@ -18,12 +18,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libxml2-dev \
     libxslt1-dev \
     nginx \
-    python2.7 \
-    python-dev \
-    python-pip \
-    python-setuptools \
+    python3-dev \
+    python3-pip \
+    python3-setuptools \
     sudo \
     vim \
+    && ln -s /usr/bin/python3 /usr/bin/python \
     && python -m pip install --upgrade pip \
     && python -m pip install --upgrade setuptools \
     && python -m pip install --upgrade uwsgi \

bin/setup_notifications.py

@@ -10,6 +10,8 @@
 def setup(host, user, password, database):
     engine = create_engine(
         "postgres://{user}:{password}@{host}/{database}".format(
-            user=user, host=host, password=password, database=database))
+            user=user, host=host, password=password, database=database
+        )
+    )
     Base.metadata.drop_all(engine)
     Base.metadata.create_all(engine)

bin/setup_psqlgraph.py

@@ -6,57 +6,69 @@
 from psqlgraph import create_all, Node, Edge
 
 
-def try_drop_test_data(user, database, root_user='postgres', host=''):
+def try_drop_test_data(user, database, root_user="postgres", host=""):
 
-    print('Dropping old test data')
+    print("Dropping old test data")
 
-    engine = create_engine("postgres://{user}@{host}/postgres".format(
-        user=root_user, host=host))
+    engine = create_engine(
+        "postgres://{user}@{host}/postgres".format(user=root_user, host=host)
+    )
 
     conn = engine.connect()
     conn.execute("commit")
 
     try:
         create_stmt = 'DROP DATABASE "{database}"'.format(database=database)
         conn.execute(create_stmt)
-    except Exception, msg:
+    except Exception as msg:
         logging.warn("Unable to drop test data:" + str(msg))
 
     conn.close()
 
 
-def setup_database(user, password, database, root_user='postgres',
-                   host='', no_drop=False, no_user=False):
+def setup_database(
+    user,
+    password,
+    database,
+    root_user="postgres",
+    host="",
+    no_drop=False,
+    no_user=False,
+):
     """
     setup the user and database
     """
-    print('Setting up test database')
+    print("Setting up test database")
 
     if not no_drop:
         try_drop_test_data(user, database)
 
-    engine = create_engine("postgres://{user}@{host}/postgres".format(
-        user=root_user, host=host))
+    engine = create_engine(
+        "postgres://{user}@{host}/postgres".format(user=root_user, host=host)
+    )
     conn = engine.connect()
     conn.execute("commit")
 
     create_stmt = 'CREATE DATABASE "{database}"'.format(database=database)
     try:
         conn.execute(create_stmt)
-    except Exception, msg:
-        logging.warn('Unable to create database: {}'.format(msg))
+    except Exception as msg:
+        logging.warn("Unable to create database: {}".format(msg))
 
     if not no_user:
         try:
             user_stmt = "CREATE USER {user} WITH PASSWORD '{password}'".format(
-                user=user, password=password)
+                user=user, password=password
+            )
             conn.execute(user_stmt)
 
-            perm_stmt = 'GRANT ALL PRIVILEGES ON DATABASE {database} to {password}'\
-                        ''.format(database=database, password=password)
+            perm_stmt = (
+                "GRANT ALL PRIVILEGES ON DATABASE {database} to {password}"
+                "".format(database=database, password=password)
+            )
             conn.execute(perm_stmt)
             conn.execute("commit")
-        except Exception, msg:
+        except Exception as msg:
             logging.warn("Unable to add user:" + str(msg))
     conn.close()
 
@@ -65,57 +77,84 @@ def create_tables(host, user, password, database):
     """
     create a table
     """
-    print('Creating tables in test database')
+    print("Creating tables in test database")
 
-    engine = create_engine("postgres://{user}:{pwd}@{host}/{db}".format(
-        user=user, host=host, pwd=password, db=database))
+    engine = create_engine(
+        "postgres://{user}:{pwd}@{host}/{db}".format(
+            user=user, host=host, pwd=password, db=database
+        )
+    )
     create_all(engine)
     versioned_nodes.Base.metadata.create_all(engine)
 
 
 def create_indexes(host, user, password, database):
-    print('Creating indexes')
-    engine = create_engine("postgres://{user}:{pwd}@{host}/{db}".format(
-        user=user, host=host, pwd=password, db=database))
+    print("Creating indexes")
+    engine = create_engine(
+        "postgres://{user}:{pwd}@{host}/{db}".format(
+            user=user, host=host, pwd=password, db=database
+        )
+    )
     index = lambda t, c: ["CREATE INDEX ON {} ({})".format(t, x) for x in c]
     for scls in Node.get_subclasses():
         tablename = scls.__tablename__
-        map(engine.execute, index(
-            tablename, [
-                'node_id',
-            ]))
-        map(engine.execute, [
-            "CREATE INDEX ON {} USING gin (_sysan)".format(tablename),
-            "CREATE INDEX ON {} USING gin (_props)".format(tablename),
-            "CREATE INDEX ON {} USING gin (_sysan, _props)".format(tablename),
-        ])
+        list(map(engine.execute, index(tablename, ["node_id",])))
+        list(
+            map(
+                engine.execute,
+                [
+                    "CREATE INDEX ON {} USING gin (_sysan)".format(tablename),
+                    "CREATE INDEX ON {} USING gin (_props)".format(tablename),
+                    "CREATE INDEX ON {} USING gin (_sysan, _props)".format(tablename),
+                ],
+            )
+        )
     for scls in Edge.get_subclasses():
-        map(engine.execute, index(
-            scls.__tablename__, [
-                'src_id',
-                'dst_id',
-                'dst_id, src_id',
-            ]))
+        list(
+            map(
+                engine.execute,
+                index(scls.__tablename__, ["src_id", "dst_id", "dst_id, src_id",]),
+            )
+        )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("--host", type=str, action="store",
-                        default='localhost', help="psql-server host")
-    parser.add_argument("--user", type=str, action="store",
-                        default='test', help="psql test user")
-    parser.add_argument("--password", type=str, action="store",
-                        default='test', help="psql test password")
-    parser.add_argument("--database", type=str, action="store",
-                        default='automated_test', help="psql test database")
-    parser.add_argument("--no-drop", action="store_true",
-                        default=False, help="do not drop any data")
-    parser.add_argument("--no-user", action="store_true",
-                        default=False, help="do not create user")
+    parser.add_argument(
+        "--host", type=str, action="store", default="localhost", help="psql-server host"
+    )
+    parser.add_argument(
+        "--user", type=str, action="store", default="test", help="psql test user"
+    )
+    parser.add_argument(
+        "--password",
+        type=str,
+        action="store",
+        default="test",
+        help="psql test password",
+    )
+    parser.add_argument(
+        "--database",
+        type=str,
+        action="store",
+        default="automated_test",
+        help="psql test database",
+    )
+    parser.add_argument(
+        "--no-drop", action="store_true", default=False, help="do not drop any data"
+    )
+    parser.add_argument(
+        "--no-user", action="store_true", default=False, help="do not create user"
+    )
 
     args = parser.parse_args()
-    setup_database(args.user, args.password, args.database,
-                   no_drop=args.no_drop, no_user=args.no_user)
+    setup_database(
+        args.user,
+        args.password,
+        args.database,
+        no_drop=args.no_drop,
+        no_user=args.no_user,
+    )
     create_tables(args.host, args.user, args.password, args.database)
     create_indexes(args.host, args.user, args.password, args.database)

bin/setup_test_database.py

@@ -18,25 +18,44 @@
 )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("--host", type=str, action="store",
-                        default='localhost', help="psql-server host")
-    parser.add_argument("--user", type=str, action="store",
-                        default='test', help="psql test user")
-    parser.add_argument("--password", type=str, action="store",
-                        default='test', help="psql test password")
-    parser.add_argument("--database", type=str, action="store",
-                        default='automated_test', help="psql test database")
-    parser.add_argument("--no-drop", action="store_true",
-                        default=False, help="do not drop any data")
-    parser.add_argument("--no-user", action="store_true",
-                        default=False, help="do not create user")
+    parser.add_argument(
+        "--host", type=str, action="store", default="localhost", help="psql-server host"
+    )
+    parser.add_argument(
+        "--user", type=str, action="store", default="test", help="psql test user"
+    )
+    parser.add_argument(
+        "--password",
+        type=str,
+        action="store",
+        default="test",
+        help="psql test password",
+    )
+    parser.add_argument(
+        "--database",
+        type=str,
+        action="store",
+        default="automated_test",
+        help="psql test database",
+    )
+    parser.add_argument(
+        "--no-drop", action="store_true", default=False, help="do not drop any data"
+    )
+    parser.add_argument(
+        "--no-user", action="store_true", default=False, help="do not create user"
+    )
 
     args = parser.parse_args()
-    setup_database(args.user, args.password, args.database,
-                   no_drop=args.no_drop, no_user=args.no_user)
+    setup_database(
+        args.user,
+        args.password,
+        args.database,
+        no_drop=args.no_drop,
+        no_user=args.no_user,
+    )
     create_tables(args.host, args.user, args.password, args.database)
     create_indexes(args.host, args.user, args.password, args.database)
     create_transaction_logs_table(args.host, args.user, args.password, args.database)