Push to prod!

.gitignore

@@ -12,9 +12,8 @@ env/
 
 ### Project specifics
 bin/
-*.zip
-*.tar.gz
+build.zip
 
 ### Sensitive info
 docker-compose.yml
-
+*.secret

.travis.yml

@@ -0,0 +1,82 @@
+language: python
+
+services:
+- docker
+
+env:
+  global:
+  - PATH=$HOME/.local/bin:$PATH
+  - BUCKET_NAME=glimpsefiles
+  - TEST_FUNCTION_DIR=test/function
+  - TEST_FUNCTION_NAME=glimpseTest
+  - RUN_FUNCTION_DIR=run/function
+  - RUN_FUNCTION_NAME=glimpseRun
+  #- secure: OpAR7Te7TvxzeIMCtpDC7rTTYijrWlh9LtCB8EBYzZkcAmKUicb/Amiocl2x6EPlkFqZXcX0acuZvEbhfrZlRrhcmSTOujHkGQk/u6j2W96v5cQ2udW03XVv24Q3suP7qmANAe+T6wfKLj7FPtP/L+IkN3gVw1rzxQP2dJZMvbS9qXul8+VdwaFqsw5i6v2ys3jpFnYtkYGttXrR6LGXVQ9iv9SJkcKYXYBOs/H2upL1pRsJHbldsQqaWQ3f34Cl4jSv/+PM3YABmol94pZqaVOkItdJFtzYoL5VV9taF/uIIOykfVEwI0hMXOfm0H/BIGCgChYnNztqfM39tCeeH1QSUaF7rS8tNwa5AKJEQkDvoQyjcWHEHtQJk8VkmvRcw/YF8zOAymfbpN72L5jy+vhabreaIt14ZsABmerL2F40gyH07KMI6UYrMT6Nnz3aAqYs+2NcwEo6mKGyk9fcXA0UT4vpApexoxbLSD57b5VVvkoMMzpg9OJUu1VzI0Rqug2sgmX4dPQda3RBCL03O2TRdUL4WaGUnWOpLyhz08d7Xh5OkIo6I4uq9XtHi7Sd7D9gi6Vzp+PbQUvpwZqnXOL3hU0GvohwuQqvgKdSkQTcecdVbwtCl1qLriU6Z54nCbSOpWaIJ2Tc4wG7CMZDEaZR4cazokS6xXSCYxM3wfk=
+  #- secure: OdjTzGgDsSatEvPBuND4y9QNjusTXcJVuBApXq1p6Sa4tX/6zrqB4k1Su1z+WfvdPlWaJ81BoQjlWnhEbKvBibOEImM2Fw7nbKntYmSyLprG6s5UHfQ4pTxaF35FdpsWOxenLywmjPr/U9JZ5Qdwce53FAg3hqAeAdE/09jIwefoWHR9kP3UBJOV0WKaYvAmnBjypJufT985hRNN59DFQnVtJWikU77vp604ArX+7jGNBOg+9huhegIpqTApEJuNX2+bkwRROCx/fAYFjrff/qOPFTFJbBToPLGNLawgDlEzZ/iXwDjBxm01U/gii+ixPv/6zm6BwLOg/tzfPHEodEHde9zEgH9rd1dEC08CnqY2SJOpKBNjdKudvdZiVpNKJEye/ebOoUSh9oRCPUjfwIkQN1fzvMat51Fto81QXbq503m+jCgISoYAWaPLFUvA0ijWnfqHOzUlzIbDoDvznPtkW/1N+JfDE8VfzMJNGT7TMD38wTIhPQYU88hJe1tw0n0sf8GoFfPs5VmIu/x1Yk5aus3lu3IQW530CqKOuyHRODjfpyBrWtUTJx7Fjk8goRuaey9/gDXL4XYyNRjx2qz4UdFi+MKuezxD9EkU2Lqt3PKEAhW6zdk268rLLgSl/1PAvQomLogbERcYwzzVdVvsp0xVZcCBdIgOuu5ERCc=
+  - secure: "H4K4IjdpfjJfQEtvXHx4lcHq+ru9XfyN0HSvrFKxSgA4raFSSoW07gum1EkirLXakOag8uFstAZokCf4LyFoe0AsdvE8x7TQ7OR7i2d6Lyv4P96SL16E7f5ZItA3dNiBKt9FudWGXbaTrQ71QcubzKrI9KRBM5LOGAQnhwU2cGQzcw8YAWpDxPHKJd+ZZ/zXtNhkLo5R9ZEg6yuoJpZdiPyEWknsYjDIwdIX3wB/AAeBa179IU4DQ2HxLGsPX/CBwXZ6eJWEwVL9Eb1fmiOtJG3LhVEbdZ2Jp8RpYbowjhM9uaIsLAQLJrB6tJ5f1lAhlkC/o8bs8SCNDLNIVqI1hyqk2QYn9ztRd9lXiQiQU8zJeadJ+BU1LnDln9zmY1FszRx3qX7V2nDaCMKpuRtICEL6DTeq/cX5UypaprGlo6Nqk4u28dsnFMkNwYS+ncUR4IZlvPRbXxY/9U5jbiav3Mf2OFTGpqBtd/fS7+8uGmKKm7YFNKcIXJZEk+ekUgXmAxWiB4MWcOCfvMObMJT3oQHWmjBCazH545d9XwZv4KGeiZslboTogQTat84yw5AXszwFJ+PUuvPd1MjCxIF5l16Qmmzk/KqJtSogTI1zJLLiuwgFJ474wdYouDysV3qvv91EQVzV9S1REMAtLajLAhbeXvlhZWT9fZVKkyuDXnI="
+  - secure: "jgHbSu95AWrI7UxSsgJzCwMYpRBewJ87MkhZBfzsDHt3y0thVJHX1553JITAqPDGVv7gHwEPa6JAy0FiXT6KPiCBopGz7LStVfumFNGwd5BKL4JF1ye+Nif+X3cZoETPPrOXUb2olWXL1ldP95VrfHJtzJYjBn2dBa/GuV8zYvXRTsbLNhKMI0QnYj3UqJYqDyXmgxew4C/d3L4FyAmeyWWDmHREAV5OZLhP0pVGC90ns+m2lOTXO16HBmC3ix64bCbbS0UfE/0wS4VdN5BZhe/PA5jw52YRO1lCcM8ILhhnNSnbIykkOuscLlwoVXmz9D3NOas24/cpT0mH11mjHMGmMRwr0w9+AosH+dzAAKQU/czrmWAF0tPC0CE+9ps+O+hV9xoiI9YHi8BfCyecm9m9a0hLxu2/ahXYg6cD2P03JXphHhI7gj1LzzAZxgXipJjj4RNPzQC/yBaUO+dG4M2u5u0/9hRRAjCmm7i23Pr4r5lGveQUHDA5O34GCy1VJBBvv+xwuZlCeD6tboAaNpzqEr9UKIoRdSVvWt22OesEGdLe0pmvm8usi8Gh4Piaq/K7U4oCt6lFTFnENffRfQ13huvah3L0WHDtDgxk7x0iZMmGU2kAatRpXVnWE0E5FaNj//hZszblTUBccH2VvmM3yDg4hWwhjyr46onQ2cA="
+  - AWS_DEFAULT_REGION=us-east-1
+
+install:
+  - pip install awscli
+
+before_script:
+  - make build
+
+script:
+  - make run URL=https://google.com
+  - make update URL=https://google.com
+  - make update URL=1.1.1.1
+  - make update URL=http://letmeoutofyour.net:8081/
+  - make update URL=httpbin.org/anything/$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold
+    -w 32 | head -n 1)
+  - make update URL=httpbin.org/redirect/6
+  - make update URL=https://httpbin.org/image/png
+  - make update URL=https://httpbin.org/headers
+  - make run URL=file:///etc/passwd || echo "SAVED - Made to fail"
+  - make run URL=gibberish || echo "SAVED - Made to fail"
+  - make run URL=mjnjvfidrilyenbbvjwisyzdpycppftjombbxqrtvkcpsbxdnllqupptmhancjqimgcpsfuhzjpdkiaibkcigwyrmaajjszyxvyjekbobdzluepicwnelaaljhkqzmqdupjbhercywvvomwzfszzeptaeaiofiohixlzfwjnlgvilyklivymuknqybunftprnvotjviimcttlyjqfhdefapbpzvzugghzaisdyrmebjurqtzbbheomgmentccdekijireporxxmvqneebbrfzhvgbydzmiaewpqopnxzdsrt || echo "Made to fail"
+
+before_deploy:
+  - make pack
+
+deploy:
+  - provider: s3
+    access_key_id: "$AWS_ACCESS_KEY_ID"
+    secret_access_key: "$AWS_SECRET_ACCESS_KEY"
+    bucket: "$BUCKET_NAME"
+    region: us-east-1
+    upload_dir: "$TEST_FUNCTION_DIR"
+    local_dir: dist/function
+    skip_cleanup: true
+    on:
+      branch: build
+  - provider: s3
+    access_key_id:
+      secure: 04W7ckwzoGdRDh4pnEqwCzZmDmSFRg2lpeTfFqftFLubByBgluuCBS0+2dezJgSytW3WNhsshjKDBCeSoXdCBvnGgfv5FvH8lieL2nfY4Q0u+8NAQ/FtaoUBxdTZmHYryHfcR2zAneBcViFG9iCQtVoUCaHcvyU+ldBggC6eLXxacGxOKP6cwSjhPPIdA130PrkqpNC1XBfc09fqY61j6E8uJQz+tsndexB2t3aolsdzb+pmBpGExmSlbs6W+GVZxZ/TCFbtgYER3vyzn23yxlwemWw/vaTgK8MQK+Etm/Uxcl0bKjO6CXQM0AcchLCTCd8Ee29xjLYZbkGVTpWrBoDRft7sebM2yjKOc2c2p+Ew0ePyrFr3VTqkvyIUZhIBKvmqRvGsEBKKhFBT/2h36GC4W3BSGoX/S1Z4+YQIakd8iP99bgfsZvmiWzP4UC844re8GjP51SU3JiUxpea5jIjcXNfQO2CCxQvH79MFcURU6BA2MVq+Tae9sAGwNtIqJPJnJ03hIzQcWPE/+vpiKYgDctvV0el6MbfDa/7whtkGMbUijcHdcfOklxIxby+M6helJCBokTXfGFlVae5RIORbxkpEaiTiHbZvAvgjXeLlRop/8jwyv8aHKu3l7FedZUHbGkfoMV2ExHCwbYNu6Kx4ngUFzBYqy+KwOXvTI0o=
+    secret_access_key:
+      secure: prKdiPz2nDNb3zUpDI5xKDfb6JhFTkEkVQFw965zAoLz6j4A4+XBywjPQ10x5fuZ5lm0at680Kb7vXIaL1ofTBImg5DLs79byh6R0muRvflJ95F0QQVy6x3J80oMD9KeUOy6kTgLSwIyv3V9OADfeo4+SZWwo/x7mv4ss9hwrzxOuD+L0a6fPUUQQbIKsQ4T408wEMLUjr7sDCCaqN28VDnJACTrAL2Oj8IGHOYzXaYpin6vfdasI/ICA2ClHxXADCg3gWg02oQQiRwI51N2L33KBb9SBQDlk3/G4nkvP0m7ikAZsWy7QBpc7skEZzrb+VjblsuH+DWnCdaITbbQB0qPTb86c/rcFkY5wIMxwK7plEfUvLoncphuJBkIEbeQr5IhRt8UuFM4bx4IXOuBw7yko2RvZiXwUL9qlN//UAWueKJWeP44Vft4nvmUPlY3hKHZrO1J4ExUCzJ6HwISviCUo+zrozCh9ubL8iDxgRLz41mG1Dk2/QxWT4uclDUmGDucgId6shrAWGxlejYsSvzNOGicoC/pasl+tTl0ItX19oNNHHxDcAcjrmhKAyktOQNdlnBUB/Q1WMr4XiKKmLgB50zqV2nywaIOgMoHzOhY3g65htX18/ReFB+0SadL7bDXIv90RtYyLpimRb+ciez/FngHFCHfcyeKAS8JMWA=
+    bucket: "$BUCKET_NAME"
+    region: us-east-1
+    upload_dir: "$RUN_FUNCTION_DIR"
+    local_dir: dist/function
+    skip_cleanup: true
+    on:
+      branch: master
+
+after_deploy:
+  - if [ $TRAVIS_BRANCH == "build" ]; then
+      aws lambda update-function-code --function-name $TEST_FUNCTION_NAME --s3-bucket $BUCKET_NAME --s3-key $TEST_FUNCTION_DIR/build.zip;
+    else echo "Skipping Lambda update for $TEST_FUNCTION_NAME.";
+    fi
+  - if [ $TRAVIS_BRANCH == "master" ]; then
+      aws lambda update-function-code --function-name $RUN_FUNCTION_NAME --s3-bucket $BUCKET_NAME --s3-key $RUN_FUNCTION_DIR/build.zip;
+    else echo "Skipping Lambda update for $RUN_FUNCTION_NAME.";
+    fi
+
+notifications:
+  slack:
+    template:
+        - "Build <%{build_url}|#%{build_number}> (<%{compare_url}|%{commit}>) of *%{repository_slug}* @ %{branch} %{result} in %{duration}"
+    rooms:
+      secure: t5ThZ20j8M5RwvJQQYyN+0MYjqefgrv7OtdUYDpzpxCCB2o9lGTYmIuQrLRZY7BIA1SZS3nIiZvwudR41ibcjfF/Np5W0FntezLRmfpPCic4V8aG4F0iTINNJPK6+6KDHmKEylSjtB30lhp2Xa/4uXvrcHRbFct/+QDAD6WtjkKQvm1OFRFq7GVkWCHp5AyQRzaJaKGJ9OjWJAHaYxN+QTuubLW0PVjbeb/3b7qt3v/iqCngmynaan5xghy2P9RiMo9ll5aTSiSrtQC1s1o2uR4ISheZ5BJPT6yypemlxPUWMBe1t48JZXp2HQCm4yncgqlavwmeWffw/cN7vYYjuQeNhgWUYVI03PbbNvF/yX1a++PFtnL4Q1/6ZKo/xcQG74fVCmnVzdBQ9s3I4JybgRTyQyXpQbmRQRmZkQKojCtenycc8ShtkRyRN/H05yq1gmRd55qBxZb2rzaFmd+uKnM1NbQmPzhR7Dn1YRLPpFBFlOVYl7xlDi8ITpdxUJvjjjbgIkTn0UamYjdG1MjXWOFgi+qQfPnQ5vYWGNOxiGOQ7uGlD8pvatyzFJkPhMBSU3QNm5OB7hTlPJDwZCMmtvV+uO4itXU6dfPO1SZdtgshUi1hUNtuu7r1JsAtq7zGXaYjLhk7feNJ6z3oq6V8DVd7HQK+TkEUN6OghAFae20=

Makefile

@@ -13,8 +13,11 @@ RUN_FUNCTION_NAME = "glimpseRun"
 TEST_S3_KEY = "test/function/build.zip"
 TEST_FUNCTION_NAME = "glimpseTest"
 
+set:
+	source ./access.secret
+
 clean:
-	rm -rf build build.zip
+	rm -rf build dist
 	rm -rf __pycache__
 
 #
@@ -50,7 +53,7 @@ fetch-dependencies:
 # needed when updating the Lambda configuration in
 # docker-compose.yml or Dockerfile 
 #
-build:
+build: clean fetch-dependencies
 	docker-compose build
 
 #
@@ -75,7 +78,8 @@ pack: clean fetch-dependencies
 	cp -r lib build/.
 	pip install -r requirements.txt -t build/lib/.
 	cd build; zip -9qr build.zip .
-	cp build/build.zip .
+	mkdir -p dist/function
+	cp build/build.zip dist/function/build.zip
 	rm -rf build
 
 #

docker-compose.yml

@@ -6,5 +6,11 @@ services:
     environment:
       - PYTHONPATH=/var/task/src:/var/task/lib
       - PATH=/var/task/bin
+      - AWS_ACCESS_KEY_ID
+      - AWS_SECRET_ACCESS_KEY
+      - GLIMPSE_BUCKET_NAME=glimpsefiles
+      - GLIMPSE_SCREENSHOT_DIR=test/screenshots/  
+      - GLIMPSE_DB_TABLE=glimpsetest
+      - CONTINUOUS_INTEGRATION=true
     volumes:
       - ./src/:/var/task/src/

src/lambda_function.py

@@ -16,7 +16,7 @@ def md5_str(string):
 
 # Fail if URL matches any bad words
 def filter(url):
-    bad_words = ['file://']
+    bad_words = ['file://', 'ftp://']
     if any(word in url for word in bad_words):
         raise Exception('suspicious string found in URL')
 
@@ -34,7 +34,8 @@ def lambda_handler(event, context):
     # Filter for potentially malicious or invalid URLs
     filter(url)
 
-    if 'http' not in url:
+    protocols = ['http', 'https']
+    if not any(proto + '://' in url for proto in protocols):
         url = 'http://' + url
     check_connection(url)
 
@@ -58,7 +59,7 @@ def lambda_handler(event, context):
     db_data = db.get({'urlhash': url_hash})
 
     if db_data is None:
-        db_data = {'urlhash': url_hash, 'url': url, 'timescanned': timestamp, 'numscans': 1}
+        db_data = {'urlhash': url_hash, 'url': url, 'timescanned': timestamp, 'numscans': 0}
     else:
         exists = True
         db_data['timescanned'] = timestamp
@@ -67,14 +68,6 @@ def lambda_handler(event, context):
     s3_key = s3.get_key(remote_path)
 
     # Don't update if update==false or the parameter doesn't exist
-    #if 'update' in event.keys():
-    #    if str(event['update']).lower() != 'true':
-    #        if exists:
-    #            return return_data
-    #else:
-    #    if exists:
-    #        return return_data
-
     if 'update' not in event.keys() or str(event['update']).lower() != 'true':
         # Don't force an update
         if exists:
@@ -89,10 +82,17 @@ def lambda_handler(event, context):
 
         db_data['effectiveurl'] = glimpse.driver.current_url
         db_data['title'] = glimpse.driver.title
-        if exists:
-            db_data['numscans'] += 1
-        else:
-            db_data['numscans'] = 1
+        if db_data['title'] == '':
+            db_data['title'] = 'No title given'
+
+        # Don't need if db_data['numscans'] is set to 0 when
+        # the DB GET doesn't exist 
+        #if exists:
+        #    db_data['numscans'] += 1
+        #else:
+        #    db_data['numscans'] = 1
+        db_data['numscans'] += 1
+
         db.put(db_data)
 
         return return_data