Initial commit

.gitignore

@@ -0,0 +1,15 @@
+lib-cov
+*.seed
+*.log
+*.csv
+*.dat
+*.out
+*.pid
+*.gz
+
+pids
+logs
+results
+
+npm-debug.log
+node_modules

LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Jerry Orr
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Procfile

@@ -0,0 +1 @@
+web: node server

README.md

@@ -0,0 +1,31 @@
+sniffles
+========
+
+A simple node.js app that cleans Google Flu Trends data into a CSV format suitable for [Dash](http://thedash.com)
+
+Quickstart
+=====
+
+    # get it from GitHub
+    git clone git@github.com:jerryorr/sniffles.git
+    cd sniffles
+
+    # Fire it up
+    node server
+
+    # United States data only
+    curl http://localhost:5000
+
+    # A variety of regions
+    curl 'http://localhost:5000?region=United%20States&region=Pennsylvania&region=Maryland'
+
+Usage on Dash
+=====
+
+ 1. Deploy `sniffles` somewhere that the intewebs can see it (like Heroku)
+ 1. Go to http://thedash.com
+ 1. Sign up for an account if you don't already have one
+ 1. Create a new dashboard
+ 1. Select + > Custom > Custom Chart
+ 1. Put `http://yourhost.org:5000?region=United%20States&region=Pennsylvania&region=Maryland` in the `Data URL` field.
+ 1. Gaze in amazement at your beautiful chart.

lib/clean-intro-filter.js

@@ -0,0 +1,27 @@
+// A filter that cleans out the header content in the Google Flu Trends data,
+// returning just the CSV part
+
+var stream = require('stream')
+  , util = require('util')
+
+function CleanIntro(options) {
+  stream.Transform.call(this, options)
+}
+
+util.inherits(CleanIntro, stream.Transform)
+
+CleanIntro.prototype._transform = function (chunk, enc, cb) {
+  if (this.readingData) {
+    this.push(chunk, enc)
+  } else {
+    // Ignore all text until we find a line beginning with 'Date,''
+    var start = chunk.toString().search(/^Date,/m)
+    if (start !== -1) {
+      this.readingData = true
+      this.push(chunk.slice(start), enc)
+    }
+  }
+  cb()
+}
+
+module.exports = CleanIntro

lib/data.js

@@ -0,0 +1,44 @@
+var request = require('request')
+  , csv = require('csv-stream')
+  , util = require('util')
+  , _ = require('lodash')
+  , moment = require('moment')
+  , OutStream = require('./out-stream')
+  , CleanIntroFilter = require('./clean-intro-filter')
+
+// Returns a Stream that emits CSV records from Google Flu Trends.
+// options:
+//   - regions: an array of regions for which data should be generated.
+//     See http://www.google.org/flutrends/us/data.txt for possible values
+module.exports = function (options) {
+  options = _.extend({
+    regions: ['United States']
+  }, options)
+
+  var out = new OutStream()
+  out.write('Date,' + options.regions.join())
+
+  var earliest = moment().subtract('years', 1)
+
+  request('http://www.google.org/flutrends/us/data.txt')
+    .pipe(new CleanIntroFilter())
+    .pipe(csv.createStream({}))
+    .on('error',function(err){
+        out.emit('error', err)
+    })
+    .on('data',function(data) {
+      var date = moment(data.Date)
+
+      // Only return data from the past year
+      if (date.isAfter(earliest) || date.isSame(earliest)) {
+        out.write(data.Date + ',' + _.map(options.regions, function (region) {
+          return data[region]
+        }).join())
+      }
+    })
+    .on('end', function () {
+      out.end()
+    })
+
+  return out
+}

lib/out-stream.js

@@ -0,0 +1,29 @@
+// A simple Stream implementation that you can write to.
+//    var out = new OutStream()
+//    out.write('hey now')
+//    out.end()
+
+var stream = require('stream')
+
+var OutStream = function() {
+  stream.Transform.call(this,{objectMode: false})
+}
+
+OutStream.prototype = Object.create(
+  stream.Transform.prototype, {constructor: {value: OutStream}} )
+
+OutStream.prototype._transform = function(chunk, encoding, callback) {
+  this.push(chunk, encoding)
+  callback && callback()
+}
+
+OutStream.prototype.write = function () {
+  this._transform.apply(this, arguments)
+}
+
+OutStream.prototype.end = function () {
+  this._transform.apply(this, arguments)
+  this.emit('end')
+}
+
+module.exports = OutStream

package.json

@@ -0,0 +1,27 @@
+{
+  "name": "sniffles",
+  "version": "0.0.1",
+  "description": "Reads Google Flu Trends data and outputs just recent U.S. data.",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/jerryorr/sniffles.git"
+  },
+  "author": "Jerry Orr <jerry.orr@spiderstrategies.com>",
+  "license": "MIT",
+  "readmeFilename": "README.md",
+  "dependencies": {
+    "request": "~2.27.0",
+    "csv-stream": "~0.1.3",
+    "express": "3.x",
+    "lodash": "~2.2.1",
+    "moment": "~2.3.1"
+  },
+  "engines": {
+    "node": "0.10.x",
+    "npm": "1.2.x"
+  }
+}

server.js

@@ -0,0 +1,31 @@
+var express = require('express')
+  , data = require('./lib/data')
+  , _ = require('lodash')
+
+var app = express()
+
+app.get('/', function(req, res){
+  var options = {}
+
+  if (req.query.region) {
+    options.regions = _.isArray(req.query.region) ? req.query.region : [req.query.region]
+  }
+
+  res.setHeader('Content-Type', 'text/csv')
+
+  data(options)
+    .on('data', function (data) {
+      res.write(data)
+      res.write('\n')
+    })
+    .on('end', function (data) {
+      res.end()
+    })
+    .on('error', function (err) {
+      console.log('error: ', error)
+    })
+})
+
+var port = process.env.PORT || 5000
+app.listen(port)
+console.log('Listening on port ' + port)