Merge pull request ga4gh#2 from bmeg/master

Code Refactor

.travis.yml

@@ -0,0 +1,17 @@
+language: python
+sudo: required
+services:
+  - docker
+python:
+  - "2.7"
+os:
+  - linux
+before_install:
+  - sudo apt-get update
+  - sudo apt-get install -y golang
+install:
+  - make depends
+  - make
+script: nosetests tests
+notifications:
+  email: false

CONTRIBUTING.md

@@ -0,0 +1,73 @@
+
+DEVELOPMENT
+-----------
+
+This reference server is built in GO, serving the protocol described in 
+https://github.com/ga4gh/task-execution-schemas (note, the repo is linked as a
+git submodule, so make sure to use `git clone --recursive`)
+
+The code structure is based on two binary programs
+
+1) tes-server: Provide client API endpoint, web UI, manages work queue.
+The client facing HTTP port (by default 8000) uses a GO web router to serve static HTML
+elements and proxy API requests to a GRPC endpoint, using the GRPC gateway code 
+(https://github.com/grpc-ecosystem/grpc-gateway)
+The GRPC endpoint (usually bound to port 9090) is based on the auto-generated GO 
+protoc GRPC server. It is bound to two services: the GA4GH Task Execution Service 
+and a internal service protocol that is used by the workers to access the work queue.
+
+2) tes-worker: A client that uses the internal service protocol to contact task 
+scheduler (on port 9090) and requests jobs. It is then responsible for obtaining the 
+required input files (thus it much have code and credentials to access object store),
+run the docker container with the provided arguments, and then copy the result files 
+out to the object store.
+
+Code Structure
+--------------
+
+Main function of taskserver program (client interface and scheduler)
+src/tes-server/
+
+Main function of worker program
+src/tes-worker/
+
+Code related to the worker, include file mapper, local and swift file system clients, docker interfaces and worker thread manager
+src/tes/worker/
+
+The compiled copy of the Task Execution Schema protobuf
+src/tes/ga4gh
+
+BoltDB based implementation of the TES api as well as the scheduler API.
+src/tes/server
+
+The compiled copy of the scheduler API protobuf
+src/tes/server/proto/
+
+Python driven unit/conformance tests
+tests
+
+HTML and angular app for web view
+share
+
+
+Rebuilding Proto code
+---------------------
+First install protoc extentions (GRPC and GRPC gateway builder programs)
+```
+make depends
+```
+Rebuild auto-generated code
+```
+make proto_build
+```
+
+
+Running Tests
+-------------
+The integration testing is done using Python scripts to drive client API requests.
+To run tests:
+```
+nosetests tests
+```
+
+

Makefile

@@ -4,21 +4,21 @@ export GOPATH
 PATH := ${PATH}:$(shell pwd)/bin
 export PATH
 
-PROTO_INC= -I ./ -I $(GOPATH)/src/github.com/gengo/grpc-gateway/third_party/googleapis/
+PROTO_INC= -I ./ -I $(GOPATH)/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/
 
 server:
-	go install ga4gh-taskserver
-	go install ga4gh-worker
+	go install tes-server
+	go install tes-worker
 
 proto_build: 
 	cd task-execution-schemas/proto && protoc $(PROTO_INC) \
-		--go_out=Mgoogle/api/annotations.proto=github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/google/api,plugins=grpc:../../src/ga4gh-tasks/ \
-		--grpc-gateway_out=logtostderr=true:../../src/ga4gh-tasks/ \
+		--go_out=Mgoogle/api/annotations.proto=github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/google/api,plugins=grpc:../../src/tes/ga4gh/ \
+		--grpc-gateway_out=logtostderr=true:../../src/tes/ga4gh/ \
 		task_execution.proto
 	cd proto && protoc \
 	  $(PROTO_INC) \
 	  -I ../task-execution-schemas/proto/ \
-	  --go_out=Mtask_execution.proto=ga4gh-tasks,plugins=grpc:../src/ga4gh-server/proto \
+	  --go_out=Mtask_execution.proto=tes/ga4gh,plugins=grpc:../src/tes/server/proto \
 		task_worker.proto
 
 grpc:
@@ -27,6 +27,6 @@ grpc:
 	go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
 
 depends: grpc
-	go get -d ga4gh-taskserver/
-	go get -d ga4gh-worker/
+	go get -d tes-server/
+	go get -d tes-worker/
 

README.md

@@ -13,12 +13,12 @@ make
 
 ## Start task server
 ```
-./bin/ga4gh-taskserver
+./bin/tes-server
 ```
 
 ## Start worker
 ```
-./bin/ga4gh-worker
+./bin/tes-worker
 ```
 
 ## Get info about task execution service

bin/tes-runner.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+import json
+import time
+import requests
+import argparse
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-s", "--server", default="http://localhost:8000")
+    parser.add_argument("task")
+    args = parser.parse_args()
+
+    with open(args.task) as handle:
+        task = json.loads(handle.read())
+
+    r = requests.post("%s/v1/jobs" % (args.server), json=task)
+    data = r.json()
+    print data
+    job_id = data['value']
+
+    for i in range(10):
+        r = requests.get("%s/v1/jobs/%s" % (args.server, job_id))
+        data = r.json()
+        if data["state"] not in ['Queued', "Running"]:
+            break
+        time.sleep(1)
+    print data
+
+
+

deploy-ubuntu.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+sudo apt-get update
+sudo apt install -y golang-go
+
+git clone https://github.com/bmeg/task-execution-server.git
+git checkout scaling
+
+make depends
+make

share/app.js

@@ -0,0 +1,56 @@
+app = angular.module('TESApp', ['ngRoute'])
+
+app.controller('JobListController', function($scope, $http) {
+		"use strict";
+
+		$scope.url = "/v1/jobs";
+		$scope.tasks = [];
+
+		$scope.fetchContent = function() {
+			$http.get($scope.url).then(function(result){
+				$scope.jobs = result.data.jobs;
+			});
+		}
+
+		$scope.fetchContent();
+});
+
+app.controller('WorkerListController', function($scope, $http) {
+		"use strict";
+
+		$scope.url = "/v1/jobs-service";
+		$scope.workers = [];
+
+		$scope.fetchContent = function() {
+			$http.get($scope.url).then(function(result){
+				$scope.workers = result.data;
+			});
+		}
+
+		$scope.fetchContent();
+});
+
+app.controller('JobInfoController',
+    function($scope, $http, $routeParams) {
+        $scope.url = "/v1/jobs/" + $routeParams.job_id
+
+        $scope.job_info = {};
+        $scope.fetchContent = function() {
+            $http.get($scope.url).then(function(result){
+                $scope.job_info = result.data
+            })
+        }
+        $scope.fetchContent();
+    }
+);
+
+app.config(['$routeProvider',
+    function($routeProvider) {
+        $routeProvider.when('/', {
+           templateUrl: 'static/list.html',
+        }).
+        when('/jobs/:job_id', {
+           templateUrl: 'static/jobs.html'
+        })
+    }
+]);

share/index.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html ng-app="TESApp">
+<head>
+    <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.4.9/angular.min.js"></script>
+    <script src="//ajax.googleapis.com/ajax/libs/angularjs/1.4.9/angular-route.js"></script>
+    <script src="static/app.js"></script>
+</head>
+<body>
+<div>
+    <h1>TES</h1>
+    <hr>
+    <div ng-view></div>
+</div>
+</body>
+</html>

share/jobs.html

@@ -0,0 +1,36 @@
+<div id="job_info" ng-controller="JobInfoController">
+    <a href="/#/">&lt;---</a>
+    <h1>Job {{job_info.jobId}}</h1>
+
+    <div>{{job_info.task.name}}</div>
+    <div>{{job_info.task.description}}</div>
+
+    <div>
+      <div>Commands</div>
+      <div ng-repeat="item in job_info.task.docker" content="item">
+          <div>Container: {{item.imageName}}</div>
+          <div>Container: {{item.cmd}}</div>
+      </div>
+    </div>
+
+    <div>Requirements</div>
+    <div ng-repeat="item in job_info.task.resources" content="item">
+        {{item}}
+    </div>
+
+    <div>State: {{job_info.state}}</div>
+
+    <div>
+      <div>Logs</div>
+      <div ng-repeat="item in job_info.task.logs" content="item">
+        <div>ExitCode</div>
+        <pre>{{item.exitCode}}</pre>
+
+        <div>STDERR</div>
+        <pre>{{item.stderr}}</pre>
+
+        <div>STDOUT</div>
+        <pre>{{item.stdout}}</pre>
+      </div>
+    </div>
+</div>

share/list.html

@@ -0,0 +1,12 @@
+
+<h2>Tasks</h2>
+<div id="task_list" ng-controller="JobListController">
+    <table ng-repeat="item in jobs" content="item" width="100%">
+        <col width="20%">
+        <col width="33%">
+        <tr>
+            <td><a href="#/jobs/{{item.jobId}}">{{item.jobId}}</a></td>
+            <td>{{item.state}}</td>
+        </tr>
+    </table>
+</div>

share/tasks.html

@@ -0,0 +1,41 @@
+<div id="task_info" ng-controller="TaskInfoController">
+<a href="#/">&lt;---</a>
+    <h1>Task {{task_info.id}}</h1>
+
+    <div>Command</div>
+    <div>{{task_info.command}}</div>
+
+    <div>Depends</div>
+    <div>{{task_info.task_depends}}</div>
+
+    <div>Container</div>
+    <div>{{task_info.container}}</div>
+
+    <div>Arguments</div>
+    <div ng-repeat="item in task_info.args" content="item">
+        {{item}}
+    </div>
+
+    <div>Tags</div>
+    <div ng-repeat="item in task_info.tags" content="item">
+        {{item}}
+    </div>
+
+    <div>Requirements</div>
+    <div ng-repeat="item in task_info.requirements" content="item">
+        {{item}}
+    </div>
+
+    <div>State</div>
+    <div>{{task_info.state}}</div>
+
+    <div>Max Retries</div>
+    <div>{{task_info.max_retry}}</div>
+
+
+    <div>Jobs</div>
+    <div ng-repeat="item in task_info.status.runs" content="item">
+        <a href="#/jobs/{{item}}">{{item}}</a>
+    </div>
+
+</div>