Add initial code for the demo

README.md

@@ -0,0 +1,28 @@
+# JOTB 2018 Demo
+
+This is the demo project for the talk:
+`Monitoring your applications and services with Prometheus`.
+
+This project has three directories, each with their own README.
+
+* `todo-app`
+  This is a simple MEAN application based on [scotch-io/node-todo](https://github.com/scotch-io/node-todo).
+* `prometheus`
+  This directory contains scripts and configuration files to download and configure
+  [Prometheus](https://prometheus.io) and Alertmanager.
+* `api-client`
+  This is a small Python utility to simulate requests to the ToDo API.
+
+## How to use this repository
+
+This repository has multiple branches to show how the `todo-app` can be modified
+to export Prometheus metrics which can then be scraped by a running Prometheus instance.
+
+To follow this demo, start at branch `s0`:
+
+```
+git checkout s0
+```
+
+and follow the instructions.
+After this, move on to the next branches in turn and continue to follow the instructions for each.

api-client/.gitignore

@@ -0,0 +1 @@
+client/*

api-client/README.md

@@ -0,0 +1,19 @@
+# ToDo API client
+
+This is a utility script to simulate requests to the ToDo API.
+It will perform a variety of GET, POST, and DELETE requests to
+different endpoints.
+
+It takes a single argument, which is the URL of the ToDo API.
+This URL can be obtained using `minikube service` as shown below.
+
+It is recommended that you use a [virtualenv](https://docs.python.org/3/tutorial/venv.html)
+to run this application:
+
+```
+python3 -m venv client
+source client/bin/activate
+pip install -r requirements.txt
+
+python client.py $(minikube service demo-prom-demo --url)
+```

api-client/client.py

@@ -0,0 +1,103 @@
+import argparse
+import json
+import random
+import time
+
+import requests
+
+
+class TodoSimulator:
+    def __init__(self, host):
+        self.host = host
+        self.todos = []
+        self.todo_count = 0
+
+    def get_todos(self):
+        try:
+            r = requests.get(self.host + '/api/todos')
+            r.raise_for_status()
+            self.todos = json.loads(r.text)
+        except Exception as e:
+            print('Error retrieving todos: {error}'.format(error=e))
+
+    def get_single_todo(self):
+        if len(self.todos) == 0:
+            return
+
+        try:
+            item_to_get = random.choice(self.todos)
+            r = requests.get('{host}{route}/{id}'.format(
+                host=self.host, route='/api/todos', id=item_to_get['_id']))
+            r.raise_for_status()
+        except Exception as e:
+            print('Error retrieving todo: {error}'.format(error=e))
+
+    def create_todo(self):
+        try:
+            item = 'ToDo item #{number}'.format(number=self.todo_count)
+            r = requests.post(self.host + '/api/todos', data={'text': item})
+            r.raise_for_status()
+            self.todos = json.loads(r.text)
+            self.todo_count += 1
+        except Exception as e:
+            print('Error creating todo: {error}'.format(error=e))
+        pass
+
+    def delete_todo(self):
+        if len(self.todos) == 0:
+            return
+
+        try:
+            item_to_delete = random.choice(self.todos)
+            r = requests.delete('{host}{route}/{id}'.format(
+                host=self.host, route='/api/todos', id=item_to_delete['_id']))
+            r.raise_for_status()
+            self.todos = json.loads(r.text)
+        except Exception as e:
+            print('Error deleting todo: {error}'.format(error=e))
+
+    def delete_all(self):
+        while len(self.todos) > 0:
+            self.delete_todo()
+
+    def nonexistent(self):
+        try:
+            r = requests.get(self.host + '/doesnotexist')
+            r.raise_for_status()
+        except Exception as e:
+            print('Expected error occurred: {error}'.format(error=e))
+
+    def simulate(self, chance_boundaries):
+        while True:
+            choice = random.random()
+            if choice < chance_boundaries['get_all']:
+                # Get all todos
+                self.get_todos()
+            elif choice < chance_boundaries['get_single']:
+                # Get a specific todo
+                self.get_single_todo()
+            elif choice < chance_boundaries['create']:
+                # Create a todo
+                self.create_todo()
+            elif choice < chance_boundaries['delete']:
+                # Delete a todo
+                self.delete_todo()
+            else:
+                self.nonexistent()
+            time.sleep(0.1 * random.random())
+
+
+if __name__=="__main__":
+    parser = argparse.ArgumentParser(
+        description='Simulate requests to the ToDo API')
+    parser.add_argument('host', type=str)
+    args = parser.parse_args()
+
+    todo_sim = TodoSimulator(args.host)
+    chance_boundaries = {
+        'get_all': 0.6,
+        'get_single': 0.7,
+        'create': 0.84,
+        'delete': 0.98
+    }
+    todo_sim.simulate(chance_boundaries)

api-client/requirements.txt

@@ -0,0 +1,6 @@
+certifi==2018.4.16
+chardet==3.0.4
+idna==2.6
+pkg-resources==0.0.0
+requests==2.18.4
+urllib3==1.22

prometheus/.gitignore

@@ -0,0 +1,5 @@
+.vagrant/*
+*.log
+
+prometheus/*
+alertmanager/*

prometheus/README.md

@@ -0,0 +1,89 @@
+# Prometheus Demo
+
+This directory contains scripts to get started with a basic Prometheus and
+Alertmanager configuration.
+
+This directory includes configuration files to configure some basic
+alerting rules in Prometheus, and configure Alertmanager to sent alerts to
+a Slack channel using an [incoming webhook](https://api.slack.com/incoming-webhooks).
+To use this alerting feature, the webhook URL will need to be added to `alertmanager.yml`.
+The alerts and their statuses can still be seen within Prometheus without enabling this,
+just skip the steps to start the Alertmanager below.
+
+If you are using Linux, you can run Prometheus and Alertmanager locally, otherwise a
+[Vagrantfile](https://www.vagrantup.com/) is available.
+
+Click on the links below to choose how you would like to run the demo:
+
+* [Vagrant](#vagrant)
+* [Linux](#linux)
+
+## Vagrant
+
+```
+vagrant up
+```
+
+This will provision a virtual machine with [Prometheus]() and
+[Alertmanager]() unpacked into the home directory.
+
+Once the machine is provisioned, connect to it as follows:
+
+```
+vagrant ssh
+```
+
+### Prometheus
+You can then start Prometheus as follows:
+
+```
+cd ~/prometheus
+./prometheus --web.external-url=http://192.168.100.100:9090
+```
+
+This will start Prometheus listening on port 9090 using the configuration
+provided in `prometheus.yml`.
+This configures Prometheus to scrape its targets every 15s and configures
+it to scrape metrics from itself.
+
+To view the Prometheus UI, it can be accessed at http://192.168.99.100:9090.
+
+### Alertmanager
+To start Alertmanager, in another terminal connected to the machine, run the following:
+
+```
+cd ~/alertmanager
+./alertmanager 
+```
+
+## Linux
+
+```
+./download.sh
+```
+
+This will download [Prometheus]() and [Alertmanager]() and unpack them into the
+current directory.
+
+### Prometheus
+You can then start Prometheus as follows:
+
+```
+cd prometheus
+./prometheus --web.external-url=http://localhost:9090
+```
+
+This will start Prometheus listening on port 9090 using the configuration
+provided in `prometheus.yml`.
+This configures Prometheus to scrape its targets every 15s and configures
+it to scrape metrics from itself.
+
+To view the Prometheus UI, it can be accessed at http://localhost:9090.
+
+### Alertmanager
+To start Alertmanager, in another terminal, run the following:
+
+```
+cd alertmanager
+./alertmanager
+```

prometheus/Vagrantfile

@@ -0,0 +1,79 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "ubuntu/bionic64"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # NOTE: This will enable public access to the opened port
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine and only allow access
+  # via 127.0.0.1 to disable public access
+  # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  config.vm.network "private_network", ip: "192.168.100.100"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  # config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+  #   # Customize the amount of memory on the VM:
+  #   vb.memory = "1024"
+  # end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision "shell", privileged: false, inline: <<-SHELL
+    curl -L https://github.com/prometheus/prometheus/releases/download/v2.2.1/prometheus-2.2.1.linux-amd64.tar.gz -o prometheus.tar.gz
+    curl -L https://github.com/prometheus/alertmanager/releases/download/v0.14.0/alertmanager-0.14.0.linux-amd64.tar.gz -o alertmanager.tar.gz
+
+    mkdir alertmanager prometheus
+    tar xvf prometheus.tar.gz -C prometheus --strip-components=1
+    tar xvf alertmanager.tar.gz -C alertmanager --strip-components=1
+    rm *.tar.gz
+  SHELL
+
+  config.vm.provision "file", source: "prometheus.yml", destination: "prometheus/prometheus.yml"
+  config.vm.provision "file", source: "alert_rules.yml", destination: "prometheus/alert_rules.yml"
+  config.vm.provision "file", source: "alertmanager.yml", destination: "alertmanager/alertmanager.yml"
+end

prometheus/alert_rules.yml

@@ -0,0 +1,24 @@
+groups:
+  - name: APIAlerts
+    rules:
+      - alert: ErrorRate
+        expr: sum(rate(http_request_duration_ms_count{code="500"}[5m])) / sum(rate(http_request_duration_ms_count[5m])) > 0.1
+        for: 1m
+        labels:
+          severity: alert
+        annotations:
+          description: 'Percentage of 500 responses from API is over 10%: {{ $value }}'
+          summary: 'High percentage of 500 responses'
+
+  # The following is a rule that is guaranteed to fire.
+  # This is just to check that the configuration is working correctly.
+  # - name: DummyRule
+  #   rules:
+  #     - alert: GoRoutines
+  #       expr: sum(go_goroutines) > 0
+  #       for: 1m
+  #       labels:
+  #         severity: alert
+  #       annotations:
+  #         description: 'Test alert works'
+  #         summary: 'Test alert works'

prometheus/alertmanager.yml

@@ -0,0 +1,29 @@
+global:
+  # Add Slack API URL here
+  slack_api_url: ''
+
+# The root route on which each incoming alert enters.
+route:
+  # When a new group of alerts is created by an incoming alert, wait at
+  # least 'group_wait' to send the initial notification.
+  # This way ensures that you get multiple alerts for the same group that start
+  # firing shortly after another are batched together on the first 
+  # notification.
+  group_wait: 10s
+
+  # When the first notification was sent, wait 'group_interval' to send a batch
+  # of new alerts that started firing for that group.
+  group_interval: 2m
+
+  # If an alert has successfully been sent, wait 'repeat_interval' to
+  # resend them.
+  repeat_interval: 3h 
+
+  # A default receiver
+  receiver: slack-receiver
+
+receivers:
+- name: slack-receiver
+  slack_configs:
+    - send_resolved: true
+      text: "Summary: {{ .CommonAnnotations.summary }}\nDescription: {{ .CommonAnnotations.description }}"