ddcc

#!/usr/bin/env python

import logging, sys, optparse, types, subprocess, json, os
from time import sleep
from collections import defaultdict
from os.path import join, basename, dirname, isfile, abspath, expanduser, isdir
from subprocess import Popen, PIPE

# === command line interface, options and help ===
parser = optparse.OptionParser("""usage: %prog [options] command arguments - decentral data coordination center, the command line interface for the cancer knowledger

commands:

submit     - submit a JSON meta data file and attached VCF files via IPFS
startNode  - connect to the private ethereum network, the knowledger block chain. Start IPFS.
register   - register an IPFS submission on the block chain

list       - list all submitted datasets of all stewards that are in the block chain
steward    - run a steward node: mirror all block-chain datasets locally with IPFS

examples:

%prog submit ALL/submission.json - submit submission.json and all files it references to IPFS
%prog startNode                  - connnect to the ethereum block chain and IPFS
%prog register QmemBUVjsYonnibtq6Jxi8yuyxmyhEuRAZQaLbZfkv1K9X  - register the submission in the block chain
%prog list                       - check if the submission has been accepted by the block chain

""")

parser.add_option("-d", "--debug", dest="debug", action="store_true", help="show debug messages")
#parser.add_option("-e", "--email", dest="email", action="store", help="email address of submitter")
#parser.add_option("-n", "--name", dest="name", action="store", help="real name of submitter")
#parser.add_option("-i", "--institution", dest="institution", action="store", help="institution of submitter")
#parser.add_option("-f", "--file", dest="file", action="store", help="run on file") 
#parser.add_option("", "--test", dest="test", action="store_true", help="do something") 
(options, args) = parser.parse_args()

if options.debug:
    logging.basicConfig(level=logging.DEBUG)
else:
    logging.basicConfig(level=logging.INFO)

# ==== globals   =====

# a lot of data is inlined, to keep the need for external files very low

gethDataDir = os.environ.get("ddccDir", ".")
logging.debug("geth data dir: %s" % gethDataDir)

gethIpcPath = "ipc:/"+abspath(join(gethDataDir, "geth.ipc"))

contractAddress = "0xb315ab588d79fa70fd9c496c050c848c8482893b"

sendVarJs = \
"""
abi = [{"constant":false,"inputs":[{"name":"_varDesc","type":"bytes"}],"name":"addSubmission","outputs":[],"type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"_from","type":"address"},{"indexed":false,"name":"hashId","type":"bytes"}],"name":"VarLog","type":"event"}];
var contract = web3.eth.contract(abi); 
var publicVarStore = contract.at('"""+contractAddress+"""');
publicVarStore.addSubmission.sendTransaction("%s", {from:eth.accounts[0]});
"""

customGenesisStr = \
"""
{
    "nonce": "0x0000000000000042",
    "timestamp": "0x0",
    "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
    "extraData": "0x0",
    "gasLimit": "0x10000000",
    "difficulty": "0x400",
    "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000",
    "coinbase": "0x3333333333333333333333333333333333333333",
    "alloc": {
    "21318dd849af1fb858cd6068897f04e25e8b7f9d" :  { "balance": "10000000000000000000000000000" } 
    }
}
"""
# ==== functions =====

def runCommand(cmd, ignoreErrors=False, verbose=False, shell=False):
    """ run command in shell, exit if not successful """
    #if type(cmd)==types.ListType:
        #cmd = " ".join(cmd)
    msg = "Running command: %s" % " ".join(cmd)
    logging.debug(msg)
    if verbose:
        logging.info(msg)

    assert(type(cmd)==types.ListType)
    ret = subprocess.call(cmd, shell=shell)
    cmd = " ".join(cmd) # for debug output

    if ret!=0:
        if ignoreErrors:
            logging.info("Could not run command %s, retcode %s" % (cmd, str(ret)))
            return None
        else:
            raise Exception("Could not run command (Exitcode %d): %s" % (ret, cmd))
    return ret

def execOutput(cmd):
    " run command and return output "
    output = subprocess.check_output(cmd)
    return output

def ipfsAdd(path):
    " add file to ipfs and return hash "
    if not isfile(path):
        print "Error: %s does not exist" % path
        sys.exit(1)
    print "Adding %s to ipfs" % path
    cmd = ["ipfs", "add", "-q", path]
    ipfsHash = execOutput(cmd).strip()
    return ipfsHash

def submitToIpfs(jsonFname):
    """ submit a json file and all referenced files to IPFS """
    sampleMeta = json.load(open(jsonFname))

    #if not options.name or not options.email or not options.institution:
        #print "Please provide a value for the --email, --name and --institution options. -h for help."
        #sys.exit(1)

    # go over meta data. Along the way, add files to ipfs and hashes to a new dict
    sampleMetaSub = {}
    for sampleId, sampleData in sampleMeta.iteritems():
        newSample = {}
        for key, val in sampleData.iteritems():
            if val=="":
                continue
            if key.endswith("_filename"):
                fname = join(dirname(jsonFname), sampleData[key])
                fileHash = ipfsAdd(fname)
                print "file hash: %s" % fileHash
                newSample[key.replace("_filename", "_ipfs_hash")] = fileHash
            newSample[key] = val

        sampleMetaSub[sampleId] = newSample

    #submitMeta = {}
    #submitMeta["submission_filename"] = basename(jsonFname)
    #submitMeta["submitter_email"] = options.email
    #submitMeta["submitter_institution"] = options.institution
    #submitMeta["submitter_name"] = options.name
    #submitMeta["sampleData"] = sampleMetaSub

    subJsonFname = jsonFname+".submitted"
    json.dump(sampleMetaSub, open(subJsonFname, "w"), sort_keys=True, indent=4, separators=(',', ': '))
    print "Submission info JSON written to %s" % subJsonFname
    jsonHash = ipfsAdd(subJsonFname)

    subIdFname = jsonFname+".submissionId"
    open(subIdFname, "w").write(jsonHash)
    print "Submission ID written to %s" % subIdFname

    print "IPFS submission OK. Submission ID is %s" % jsonHash

def initGeth():
    " start geth on the local machine, connect to the genome network and start mining"
    if not isdir(gethDataDir):
        os.makedirs(gethDataDir)
        logging.info("Created directory %s" % gethDataDir)

    if not isdir(join(gethDataDir, "chainData")):
        logging.info("Initializing a geth account")
        cmd = ["geth", "--datadir", gethDataDir, "account", "new"]
        runCommand(cmd)

def gethUnlockAccount():
    logging.info("Reading password from ~/.ddccPwd")
    pwd = open(expanduser("~/.ddccPwd")).read().strip()

    runGethCmd("personal.unlockAccount(eth.accounts[0], '%s');" % pwd)
    #cmd = ["geth", "--exec", """ "personal.unlockAccount(eth.accounts[0], '%s');" """ % pwd, "attach", gethIpcPath]
    #runCommand(cmd)
    logging.info( "Unlocked geth account" )


def startNode():
    " start geth on the local machine, connect to the genome network, unlock account and start mining"
    cgFname = join(gethDataDir, "customGenesis.json")
    if not isfile(cgFname):
        cgfh = open(cgFname, "w").write(customGenesisStr)

    # mine with a single thread
    logging.info( "Starting geth and mining...")
    cmd = ["geth", "--genesis", cgFname, "--datadir", gethDataDir, \
            "--identity","ddccNode", "--networkid", "93324", "--autodag", \
            "--nat",  "any", "--mine", "1", "2>", join(gethDataDir, "geth.log"), "&"]

    ret = os.system(" ".join(cmd))
    if ret!=0:
        print "Could not run command %s" % " ".join(cmd)
        sys.exit(1)

    sleep(2) # give some time for startup

    gethUnlockAccount()

    cmd = ["ipfs", "daemon", "--init", "&"]
    runCommand(cmd, shell=True) # spawn a process
    logging.info("Started IPFS daemon")

def runGethCmd(jsStr):
    " execute a js script through geth attach and return result "
    cmd = ["geth", "--exec", jsStr, "attach", gethIpcPath]
    logging.debug("running command: %s" % " ".join(cmd))
    ret = subprocess.check_output(cmd)
    logging.debug("command output: %s" % ret)
    return ret

def registerSubmission(ipfsHash):
    " connect to running geth and send an ethereum transaction with the hashId. Return transaction ID. "
    gethUnlockAccount()

    jsCmd = sendVarJs % ipfsHash
    tid = runGethCmd(jsCmd)
    tid = tid.replace('"','')
    print "Transaction ID: %s" % tid
    return tid

def iterLog():
    "yield (addr, hash) tuples received from block chain"
    jsCode = '''var filter = web3.eth.filter({'fromBlock' : 0, address : "%s"}); var f = filter.watch(function (error, log) { var v = console.log(log.data); });''' % contractAddress
    cmd = ["geth", "--exec",  jsCode, "attach", gethIpcPath]
    logging.debug("Running %s" % " ".join(cmd))
    pipe = Popen(cmd, stdout=PIPE)
    for l in pipe.stdout:
        if len(l)<=10:
            continue
        if "unable" in l:
            continue
        l = l.strip().strip("0x")
        fromAddr = l[:40].lstrip("00")
        fileHash = l[137:].strip("00")
        hashStr = fileHash.decode("hex")
        yield fromAddr, hashStr[1:]

def listSubmissions():
    " list all submissions ever sent to the contract on the ethereum network "
    print "from	ipfsHash"
    for fromAddr, ipfsHash in iterLog():
        print fromAddr, ipfsHash

def runSteward():
    " "
    for fromAddr, ipfsHash in iterLog():
        cmd = ["ipfs", "pin", ipfsHash]
        runCommand(cmd)
# ----------- main --------------
def main():
    if len(args)==0:
        parser.print_help()
        sys.exit(0)

    command = args[0]

    if command=="submit":
        jsonFname = args[1]
        submitToIpfs(jsonFname)
    elif command=="initGeth":
        initGeth()
    elif command=="startNode":
        startNode()
    elif command=="register":
        ipfsHash = args[1]
        registerSubmission(ipfsHash)
    elif command=="list":
        listSubmissions()
    elif command=="steward":
        runSteward()


main()