run

#!/usr/bin/ruby

# This is the main entry point for running all SEMPRE programs.  See
# fig/lib/execrunner.rb for more documentation for how commands are generated.
# There are a bunch of modes that this script can be invoked with, which
# loosely correspond to the modules.

$: << 'fig/lib'
require 'execrunner'

$modes = []
def addMode(name, description, func)
  $modes << [name, description, func]
end

def codalab(dependencies=nil)
  # Set @cl=1 to run job on CodaLab
  dependencies ||= l(':fig', ':lib', ':module-classes.txt', ':libsempre')
  l(
    letDefault(:cl, 0),
    sel(:cl,
      l(),
      l('cl', 'run', dependencies, '---'),
    nil),
  nil)
end

def header(modules='core', codalabDependencies=nil)
  l(
    codalab(codalabDependencies),
    # Queuing system
    letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')),
    # Create execution directory
    letDefault(:pooldir, 1),
    sel(:pooldir, l(), 'fig/bin/qcreate'),
    # Run the Java command...
    'java',
    '-ea',
    '-Dmodules='+modules,
    # Memory size
    letDefault(:memsize, 'default'),
    sel(:memsize, {
      'tiny' => l('-Xms2G', '-Xmx4G'),
      'low' => l('-Xms5G', '-Xmx7G'),
      'default' => l('-Xms8G', '-Xmx10G'),
      'medium' => l('-Xms12G', '-Xmx14G'),
      'high' => l('-Xms20G', '-Xmx24G'),
      'higher' => l('-Xms40G', '-Xmx50G'),
      'impressive' => l('-Xms75G', '-Xmx90G'),
    }),
    # Classpath
    '-cp', 'libsempre/*:lib/*',
    # Profiling
    letDefault(:prof, 0), sel(:prof, l(), '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt'),
  nil)
end

def unbalancedTrainDevSplit
  l(o('Dataset.trainFrac', 0.8), o('Dataset.devFrac', 0.2))
end
def balancedTrainDevSplit
  l(o('Dataset.trainFrac', 0.5), o('Dataset.devFrac', 0.5))
end

def figOpts; l(selo(:pooldir, 'execDir', 'exec', '_OUTPATH_'), o('overwriteExecDir'), o('addToView', 0)) end

############################################################
# Unit tests

addMode('test', 'Run unit tests', lambda { |e|
  l(
    'java', '-ea', '-Xmx12g', '-cp', 'libsempre/*:lib/*',
    lambda { |e|
      e.key?(:sparqlserver) ? "-Dsparqlserver=http://#{e[:sparqlserver]}/sparql" : l()
    },
    'org.testng.TestNG',
    lambda { |e|
      if e[:class]
        l('-testclass', 'edu.stanford.nlp.sempre.' + e[:class])
      else
        'testng.xml'
      end
    },
    lambda { |e|
      if e[:fast]
        o('excludegroups', 'sparql,corenlp')
      else
        nil
      end
    },
  nil)
})

############################################################
# Freebase

def freebaseHeader; header('core,freebase') end

def freebaseFeatureDomains
  [
    'basicStats',
    'alignmentScores',
    'entityFeatures',
    'context',
    'skipPos',
    'joinPos',
    'wordSim',
    'lexAlign',
    'tokenMatch',
    'rule',
    'opCount',
    'constant',
    'denotation',
    'whType',
    'span',
    'derivRank',
    'lemmaAndBinaries',
  nil].compact
end

def sparqlOpts
  l(
    required(:sparqlserver, 'host:port of the Sparql server'), # Example: jonsson:3093, etc.
    o('SparqlExecutor.endpointUrl', lambda{|e| 'http://'+e[:sparqlserver]+'/sparql'}),
  nil)
end

def freebaseOpts
  l(
    figOpts,
    sparqlOpts,

    # Features
    o('FeatureExtractor.featureDomains', *freebaseFeatureDomains),
    o('Builder.executor', 'freebase.SparqlExecutor'),
    o('Builder.valueEvaluator', 'freebase.FreebaseValueEvaluator'),
    o('LanguageAnalyzer.languageAnalyzer', 'corenlp.CoreNLPAnalyzer'),

    # Lexicon
    o('LexiconFn.lexiconClassName', 'edu.stanford.nlp.sempre.fbalignment.lexicons.Lexicon'),
    l( # binary
      o('BinaryLexicon.binaryLexiconFilesPath', 'lib/fb_data/7/binaryInfoStringAndAlignment.txt'),
      o('BinaryLexicon.keyToSortBy', 'Intersection_size_typed'),
    nil),
    o('UnaryLexicon.unaryLexiconFilePath','lib/fb_data/7/unaryInfoStringAndAlignment.txt'), # unary
    o('EntityLexicon.entityPopularityPath','lib/fb_data/7/entityPopularity.txt'), # entity
    #Jonathan - added this 3/5/2015
    o('TypeInference.typeLookup','freebase.FreebaseTypeLookup'),
    o('FreebaseSearch.cachePath', '/u/nlp/data/semparse/scr/cache/fbsearch/1.cache'),
  nil)
end

def cachePaths(lexiconFnCachePath, sparqlExecutorCachePath)
  l(
    required(:cacheserver, 'none (don\'t cache to disk), local (write to local file), or <hostname>:<port> (hit the cacheserver)'),
    lambda { |e|
      cacheserver = e[:cacheserver]
      cacheserver = 'jonsson:4000' if cacheserver == 'remote' # Default
      case cacheserver
      when 'none' then l()
      when 'local' then l( # Use files directly - don't run more than one job that does this!
        o('Lexicon.cachePath', 'LexiconFn.cache'),
        o('SparqlExecutor.cachePath', 'SparqlExecutor.cache'),
        o('FreebaseSearch.cachePath', 'FreebaseSearch.cache'),
      nil)
      else l(
        o('Lexicon.cachePath', cacheserver+':/u/nlp/data/semparse/cache/'+lexiconFnCachePath),
        o('SparqlExecutor.cachePath', cacheserver+':/u/nlp/data/semparse/cache/'+sparqlExecutorCachePath),
        o('FreebaseSearch.cachePath', cacheserver+':/u/nlp/data/semparse/cache/fbsearch/1.cache'),
        # Read-only
        o('EntityLexicon.mid2idPath', cacheserver+':/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map'),
        o('FreebaseTypeLookup.entityTypesPath', cacheserver+':/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized.en-types'),
      nil)
      end
    },
  nil)
end

# tag is either "free917" or "webquestions"
def emnlp2013AblationExperiments(tag)
  l(
    letDefault(:ablation, 0),
    # Ablation experiments (EMNLP)
    sel(:ablation,
      l(), # (0) Just run things normally
      selo(nil, 'Parser.beamSize', 200, 50, 10), # (1) Vary beam size
      selo(nil, 'Dataset.trainFrac', 0.1, 0.2, 0.4, 0.6), # (2) Vary training set size
      sel(nil, # (3) Structural: only do join or only do bridge
        o('Grammar.tags', l(tag, 'join')),
        o('Grammar.tags', l(tag, 'bridge')),
        o('Grammar.tags', l(tag, 'inject')),
      nil),
      sel(nil, # (4) Features
        o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains+['lexAlign'])), # +lexAlign
        o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains+['lexAlign']-['alignmentScores'])), # +lexAlign -alignmentScores
        o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains-['denotation'])), # -denotation
        o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains-['skipPos', 'joinPos'])), # -syntax features (skipPos, joinPos)
      nil),
      #o('Builder.executor', 'FormulaMatchExecutor'), # (6) train on logical forms (doesn't really work well)
    nil),

    letDefault(:split, 0), selo(:split, 'Dataset.splitRandom', 1, 2, 3),
  nil)
end

def free917
  l( # Data
    letDefault(:data, 0),
    sel(:data,
      l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json'), unbalancedTrainDevSplit), # (0) train 0.8, dev 0.2
      l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json', 'test,data/free917.test.examples.canonicalized.json')), # (1) Don't run on test yet!
    nil),

    # Grammar
    o('Grammar.inPaths', 'freebase/data/emnlp2013.grammar'),
    o('Parser.beamSize', 500), 

    emnlp2013AblationExperiments('free917'),

    # lexicon index
    letDefault(:lucene, 0),
    sel(:lucene,
      l(
        o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/free917/'),
        cachePaths('10/LexiconFn.cache', '10/SparqlExecutor.cache'),
        o('Grammar.tags', 'free917', 'bridge', 'join', 'inject', 'exact'),
      nil),
      l( # With entity disambiguation - currently too crappy
        o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'),
        cachePaths('4/LexiconFn.cache', '4/SparqlExecutor.cache'),
        o('Grammar.tags', 'free917', 'bridge', 'join', 'inject', 'inexact'),
      nil),
    nil),
    # Use binary predicate features (overfits on free917)
    o('BridgeFn.filterBadDomain',false),
    # Learning
    o('Learner.maxTrainIters', 6),
  nil)
end

def webquestions
  l(
    # Data
    letDefault(:data, 0),
    sel(:data,
      l( # Webquestions (dev) [EMNLP final JSON]
        o('Dataset.inPaths',
          'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json'),
        unbalancedTrainDevSplit,
      nil),
      l( # Webquestions (test) [EMNLP final JSON]
        o('Dataset.inPaths',
          'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json',
          'test,lib/data/webquestions/dataset_11/webquestions.examples.test.json'),
      nil),
    nil),

    # Grammar
    letDefault(:grammar, 1),
    sel(:grammar, l(), l(o('Grammar.inPaths', 'freebase/data/emnlp2013.grammar'))),

    o('Parser.beamSize', 200), # {07/03/13}: WebQuestions is too slow to run with default 500, so set to 200 for now...

    # Caching
    letDefault(:entitysearch, 0),
    sel(:entitysearch,  # Used for EMNLP 2013
      l(
        cachePaths('lucene/0.cache', 'sparql/3.cache'),
        o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'),
        o('LexiconFn.maxEntityEntries',10),
        o('Grammar.tags', 'webquestions', 'bridge', 'join', 'inject','inexact'), # specify also strategy
      nil),
    nil),

    # Learning
    o('Learner.maxTrainIters', 3),

    # Use binary predicate features (overfits on free917)
    o('BridgeFn.useBinaryPredicateFeatures', true),
    o('BridgeFn.filterBadDomain',true),
    letDefault(:split, 0), selo(:split, 'Dataset.splitRandom', 1,2,3),
  nil)
end


addMode('freebase', 'Freebase (for EMNLP 2013, ACL 2014, TACL 2014)', lambda { |e| l(
  letDefault(:train, 0),
  letDefault(:interact, 0),

  # nlpsub: for running commands on PBS
  letDefault(:nlpsub, 0),
  sel(:nlpsub,
    l(),
    l('nlpsub', '-d/scr/yonatan/sandbox/blackhole', '-nyonatan', '-c3'),
    l('nlpsub', '-d/scr/yonatan/sandbox/blackhole', '-nyonatan', '-qjag', '-c3'),
    l('nlpsub', '-d/scr/yonatan/sandbox/blackhole', '-nyonatan', '-qjohn', '-c3'),
  nil),
  sel(:interact, l()),
  freebaseHeader,
  'edu.stanford.nlp.sempre.Main',
  freebaseOpts,

  # Dataset
  sel(:domain, {
    'webquestions' => webquestions,
    'free917' => free917,
  }),


  sel(:interact, l(), l(
    # After training, run interact, which loads up a set of parameters and
    # puts you in a prompt.
    o('Dataset.inPaths'),
    o('Learner.maxTrainIters', 0),
    required(:load, 'none or exec number (e.g., 15) to load'),
    lambda { |e|
      if e[:load] == 'none' then
        l()
      else
        execPath = "lib/models/#{e[:load]}.exec"
        l(
          o('Builder.inParamsPath', execPath+'/params'),
          o('Grammar.inPaths', execPath+'/grammar'),
          o('Master.logPath', lambda{|e| 'state/' + e[:domain] + '.log'}),
          o('Master.newExamplesPath', lambda{|e| 'state/' + e[:domain] + '.examples'}),
          o('Master.onlineLearnExamples', true),
          # Make sure features are set properly!
        nil)
      end
    },
    o('Main.interactive'),
  nil))
) })

addMode('cacheserver', 'Start the general-purpose cache server that serves files with key-value maps', lambda { |e|
  l(
    'java', '-Xmx36g', '-ea', '-cp', 'libsempre/*:lib/fig.jar',
    'edu.stanford.nlp.sempre.cache.StringCacheServer',
    letDefault(:port, 4000),
    lambda { |e| o('port', e[:port]) },

    letDefault(:cachetype, 0),
    sel(:cachetype,
      l(
        o('FileStringCache.appendMode'),
        o('FileStringCache.capacity', 35 * 1024),
        o('FileStringCache.flushFrequency', 2147483647),
      nil),
      l(
        o('FileStringCache.appendMode',false),
        o('FileStringCache.capacity', 1 * 1024),
        o('FileStringCache.flushFrequency', 100000),
      nil),
    nil),
  nil)
})

############################################################
# Freebase RDF database (for building SPARQL database)

# Scratch directory
def scrOptions
  letDefault(:scr, '/u/nlp/data/semparse/rdf/scr/' + `hostname | cut -f 1 -d .`.chomp)
end

addMode('filterfreebase', '(1) Filter RDF Freebase dump (do this once) [takes about 1 hour]', lambda { |e| l(
  scrOptions,
  l(
    'fig/bin/qcreate', o('statePath', lambda{|e| e[:scr] + '/state'}),
    'java', '-ea', '-Xmx20g', '-cp', 'libsempre/*:lib/*',
    'edu.stanford.nlp.sempre.freebase.FilterFreebase',
    o('inPath', '/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized'),
    sel(:keep, {
      'all' => o('keepAllProperties'),
      'geo' => l(
        o('keepTypesPaths', 'data/geo.types'),
        o('keepPropertiesPath', 'data/geo.properties'),
        o('keepGeneralPropertiesOnlyForSeenEntities', true),
      nil),
    }),
    o('execDir', '_OUTPATH_'), o('overwriteExecDir'),
  nil),
nil) })

addMode('sparqlserver', '(2) Start the SPARQL server [do this every time]', lambda { |e| l(
  scrOptions,
  required(:exec),
  sel(nil,
    l(
      'freebase/scripts/virtuoso', 'start',
      lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/vdb'}, # DB directory
      lambda{|e| 3000+e[:exec]}, # port
    nil),
    # Give everyone permissions so that anyone can kill the server if needed.
    l(
      'chmod', '-R', 'og=u',
      lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/vdb'}, # DB directory
    nil),
    # To stop the server: freebase/scripts/virtuoso stop 3093
  nil),
nil) })

# (3) Index the filtered RDF dump [takes 48 hours]
addMode('indexfreebase', '(3) Index the filtered RDF dump [takes 48 hours for Freebase]', lambda { |e| l(
  letDefault(:stage, nil),
  scrOptions,
  required(:exec),
  sel(:stage,
    l(
      'scripts/virtuoso', 'add',
      lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/0.ttl'}, # ttl file
      lambda{|e| 3000+e[:exec]}, # port
      lambda{|e| e[:offset] || 0}, # offset
    nil),
    l(
      'scripts/extract-freebase-schema.rb',
      lambda{|e| 'http://localhost:'+(3000+e[:exec]).to_s+'/sparql'}, # port
      lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/schema.ttl'},
    nil),
  nil),
nil) })

addMode('convertfree917', 'Convert the Free917 dataset', lambda { |e| l(
  'java', '-ea', '-Xmx15g',
  '-cp', 'libsempre/*:lib/*',
  'edu.stanford.nlp.sempre.freebase.Free917Converter',
  o('inDir','/u/nlp/data/semparse/yates/final-dataset-acl-2013-all/'),
  o('outDir','data/free917_convert/'),
  o('entityInfoFile','/user/joberant/scr/fb_data/3/entityInfo.txt'),
  o('cvtFile','lib/fb_data/2/Cvts.txt'),
  o('midToIdFile','/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map'),
nil) })

addMode('query', 'Query a single logical form or SPARQL', lambda { |e| l(
  codalab,
  'java', '-ea',
  '-cp', 'libsempre/*:lib/*',
  'edu.stanford.nlp.sempre.freebase.SparqlExecutor',
  sparqlOpts,
nil) })

############################################################


# Just start a simple interactive shell to try out SEMPRE commands
addMode('simple', 'Simple shell', lambda { |e| l(
  codalab, 'java', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
  o('Main.interactive'),
nil) })

addMode('simple-sparql', 'Simple shell for querying SPARQL', lambda { |e| l(
  codalab, 'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
  o('executor', 'freebase.SparqlExecutor'),
  sparqlOpts,
  o('Main.interactive'),
nil) })

addMode('simple-lambdadcs', 'Simple shell for querying with the LambdaDCSExecutor', lambda { |e| l(
  codalab, 'java', '-Dmodules=core,tables,corenlp', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
  o('executor', 'tables.lambdadcs.LambdaDCSExecutor'),
  o('FeatureExtractor.featureDomains', 'denotation lexAlign joinPos skipPos'.split),
  o('LanguageAnalyzer.languageAnalyzer', 'corenlp.CoreNLPAnalyzer'),
  o('Main.interactive'),
nil) })

addMode('simple-freebase', 'Simple shell for using Freebase', lambda { |e| l(
  'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
  o('executor', 'freebase.SparqlExecutor'),
  letDefault(:sparqlserver, 'freebase.cloudapp.net:3093'),
  letDefault(:cacheserver, 'freebase.cloudapp.net:4000'),
  sparqlOpts,
  # Set up Freebase search for entities
  # Assume run following on the server (read-only and capacity are important!)
  #   ./run @mode=cacheserver -readOnly -capacity MAX -basePath lib/fb_data
  o('FreebaseSearch.cachePath', 'FreebaseSearch.cache'),
  o('EntityLexicon.mid2idPath', lambda { |e| e[:cacheserver] + ':freebase-rdf-2013-06-09-00-00.canonical-id-map.gz' }),
  o('TypeInference.typeLookup', 'freebase.FreebaseTypeLookup'),
  o('FreebaseTypeLookup.entityTypesPath', lambda { |e| e[:cacheserver] + ':freebase-rdf-2013-06-09-00-00.canonicalized.en-types.gz' }),
  o('EntityLexicon.maxEntries', 2),
  o('FeatureExtractor.featureDomains', 'rule'),
  o('Parser.coarsePrune'),
  o('JoinFn.typeInference'),
  o('UnaryLexicon.unaryLexiconFilePath', '/dev/null'),
  o('BinaryLexicon.binaryLexiconFilesPath', '/dev/null'),
  #o('JoinFn.showTypeCheckFailures'),  # Use this to debug
  o('Grammar.inPaths', 'freebase/data/demo1.grammar'),  # Override with your own custom grammar
  o('SparqlExecutor.returnTable'),
  #o('SparqlExecutor.includeSupportingInfo'),  # Show full information
  o('Main.interactive'),
nil) })

addMode('simple-freebase-nocache', 'Simple shell for using Freebase (without a cache server)', lambda { |e| l(
  'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
  o('executor', 'freebase.SparqlExecutor'),
  letDefault(:sparqlserver, 'freebase.cloudapp.net:3093'),
  sparqlOpts,
  o('FeatureExtractor.featureDomains', 'rule'),
  o('Parser.coarsePrune'),
  o('JoinFn.typeInference'),
  o('UnaryLexicon.unaryLexiconFilePath', '/dev/null'),
  o('BinaryLexicon.binaryLexiconFilesPath', '/dev/null'),
  #o('JoinFn.showTypeCheckFailures'),  # Use this to debug
  o('Grammar.inPaths', 'freebase/data/demo1.grammar'),  # Override with your own custom grammar
  #o('SparqlExecutor.includeSupportingInfo'),  # Show full information
  o('Main.interactive'),
nil) })


############################################################
# {2014-12-27} [Percy]: Overnight semantic parsing
def overnightFeatureDomains
  [
    'match',
    'ppdb',
    'skip-bigram',
    'root',
    'alignment',
    'lexical',
    'root_lexical',
    'lf',
    'simpleworld',
  nil].compact
end

addMode('overnight', 'Overnight semantic parsing', l(
  header('core,freebase,overnight'),
  'edu.stanford.nlp.sempre.Main',
  figOpts,
  o('JavaExecutor.convertNumberValues', false),
  o('useAnchorsOnce', true),
  o('trackLocalChoices'),
  o('JoinFn.typeInference', true),
  o('Builder.parser', 'FloatingParser'),
  o('FloatingParser.executeAllDerivations', 'true'),
  o('LanguageAnalyzer', 'corenlp.CoreNLPAnalyzer'),
  o('Learner.maxTrainIters', 1),
  #o('printAllPredictions'),
  o('Derivation.showUtterance'),
  letDefault(:debug, 0),

  selo(1, 'maxExamples', 'train:10', 'train:MAX'),

  # Exact matching is needed on most simple domains
  # o('executor', 'FormulaMatchExecutor'),
  # o('Builder.valueEvaluator', 'ExactValueEvaluator'),

  # Features
  o('FeatureExtractor.featureDomains', 'denotation'), # denotation features from general feature extractor
  o('FeatureExtractor.featureComputers', 'overnight.OvernightFeatureComputer'), # 
  o('OvernightFeatureComputer.featureDomains', *overnightFeatureDomains),
  #o('initialization', 'paraphrase :: match,1', 'paraphrase :: size,-0.1', 'paraphrase :: ppdb,0.3',
  #  'paraphrase :: skip-bigram,0.8', 'paraphrase :: skip-ppdb,0.2','denotation :: error,-1000'),
  o('coarsePrune'),
  o('OvernightDerivationPruningComputer.applyHardConstraints'),
  sel(2,
    l(), # no reg
    l(o('Params.l1Reg','lazy'),o('Params.l1RegCoeff',0)),
    l(o('Params.l1Reg','lazy'),o('Params.l1RegCoeff',0.001)),
  nil),
  # Set up the domain
  required(:domain),
  o('Grammar.inPaths', lambda { |e| 'overnight/' + e[:domain] + '.grammar' }),
  o('SimpleWorld.domain', lambda { |e| e[:domain] }),
  o('PPDBModel.ppdbModelPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '-ppdb.txt' }),
  o('Dataset.trainFrac', 0.8), o('Dataset.devFrac', 0.2),
  o('FloatingParser.maxDepth', 11),
  o('Parser.beamSize', 20),
  letDefault(:alignment, 1),
  sel(:alignment, 
    o('wordAlignmentPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '.word_alignments.heuristic' }), 
    o('wordAlignmentPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '.word_alignments.berkeley' }),
  nil),
  o('phraseAlignmentPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '.phrase_alignments' }),
  o('PPDBModel.ppdbModelPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '-ppdb.txt' }),
  o('DerivationPruner.pruningComputers', ['overnight.OvernightDerivationPruningComputer']),
  o('Dataset.inPaths', 
    lambda { |e| 'train:lib/data/overnight/' + e[:domain] + '.paraphrases.train.examples' },
    lambda { |e| 'test:lib/data/overnight/' + e[:domain] + '.paraphrases.test.examples' }),
  sel(:domain, {
    'geo880' => l(
      letDefault(:data,0),
      sel(:data,
        l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives.examples')),
        l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives.examples', 'test:lib/data/overnight/geo880-train.examples')),
        l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives2.examples', 'test:lib/data/overnight/geo880-train.examples')),
        l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives.examples', 'test:lib/data/overnight/geo880-test.examples')),
        l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives2.examples', 'test:lib/data/overnight/geo880-test.examples')),
      nil),
      o('Parser.beamSize', 20),
      o('initialization', 'paraphrase :: match,1', 'paraphrase :: size,-0.1', 'paraphrase :: ppdb,0.3',
          'lf :: edu.stanford.nlp.sempre.SimpleWorld.superlative& superlative,10',
          'root :: pos0=WRB&returnType=class edu.stanford.nlp.sempre.NumberValue,10'),
      o('FloatingParser.maxDepth', 11),
      o('Grammar.tags','generate','general', 'geo880'),
      o('SimpleLexicon.inPaths', 'lib/data/overnight/geo880.lexicon'),
      o('OvernightDerivationPruningComputer.usePredicateDict'),
      o('OvernightDerivationPruningComputer.predicateDictPath','lib/data/overnight/geo880.predicate.dict'),
    nil),
    'calendar' => l(
      o('Grammar.tags','generate','general'),
    nil),
    'calendarplus' => l(
      o('Grammar.tags','generate','general','geo440'),
      o('Grammar.inPaths','overnight/calendar.grammar'),
      o('SimpleWorld.domain', 'calendar'),
    nil),
    'blocks' => l(
      o('Grammar.tags','generate','general'),
    nil),
    'restaurants' => l(
      o('Grammar.tags','generate','general'),
    nil),
    'housing' => l(
      o('Grammar.tags','generate','general'),
    nil),
    'socialnetwork' => l(
      o('Grammar.tags','generate','general'),
    nil),
    'publications' => l(
      o('Grammar.tags','generate','general'),
    nil),
    'basketball' => l(
      o('Grammar.tags','generate','general'),
    nil),
    'recipes' => l(
      o('Grammar.tags','generate','general'),
    nil),
  }),
nil))

############################################################
# {5/27/15} [Ice]
addMode('tables', 'QA on HTML tables', lambda { |e| l(
  # Add @cldir=1 to use CodaLab's directory paths
  letDefault(:cldir, 0),
  # Usual header
  header('core,tables,corenlp'),
  # Select class
  letDefault(:class, 'main'),
  sel(:class, {
    'main' => 'edu.stanford.nlp.sempre.Main',
    'check' => 'edu.stanford.nlp.sempre.tables.test.DPParserChecker',
    'align' => 'edu.stanford.nlp.sempre.tables.alignment.IBMAligner',
    'dump' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedDumper',
    'load' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedLoader',
    'stats' => 'edu.stanford.nlp.sempre.tables.test.TableStatsComputer',
  }),
  # Fig parameters
  selo(:cldir, 'execDir', '_OUTPATH_', '.'),
  o('overwriteExecDir'), o('addToView', 0), o('jarFiles', 'libsempre/*'),
  sel(:cldir, l(), '>/dev/null'),
  # Set environment for table execution
  o('executor', 'tables.lambdadcs.LambdaDCSExecutor'),
  o('Builder.valueEvaluator', 'tables.TableValueEvaluator'),
  o('TargetValuePreprocessor.targetValuePreprocessor', 'tables.TableValuePreprocessor'),
  o('NumberFn.unitless'), o('NumberFn.alsoTestByConversion'),
  o('TypeInference.typeLookup', 'tables.TableTypeLookup'),
  o('JoinFn.specializedTypeCheck', false), o('JoinFn.typeInference', true),
  # Parser
  letDefault(:parser, 'floatsize'),
  sel(:parser, {
    'floatsize' => l(
      o('Builder.parser', 'FloatingParser'),
      o('useSizeInsteadOfDepth'),
      o('FloatingParser.maxDepth', 15),
    nil),
    'baseline' => l(
      o('Builder.parser', 'tables.baseline.TableBaselineParser'),
    nil),
    'dummy' => o('Builder.parser', 'tables.serialize.DummyParser'),
  }),
  o('FloatingParser.useAnchorsOnce', true),
  letDefault(:pruning, 1),
  sel(:pruning,
    l(),
    l(
      o('DerivationPruner.pruningStrategies', *tablesPruningStrategies),
      o('DerivationPruner.pruningComputers', 'tables.TableDerivationPruningComputer'),
    nil),
  nil),
  # Grammar
  tablesGrammarPaths,
  # Dataset
  letDefault(:data, 'none'),
  letDefault(:unseen, 0),
  tablesDataPaths,
  # Verbosity
  o('FeatureVector.ignoreZeroWeight'),
  o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10), o('logFeaturesLimit', 10),
  letDefault(:verbose, 0),
  sel(:verbose,
    l(),
    l(
      o('showRules'),
      o('Parser.verbose', 2),
      o('JoinFn.verbose', 3),
      o('JoinFn.showTypeCheckFailures'),
    nil),
  nil),
  # Language Analyzer
  letDefault(:lang, 'corenlp'),
  sel(:lang, {
    'simple' => o('LanguageAnalyzer', 'SimpleAnalyzer'),
    'corenlp' => l(o('LanguageAnalyzer', 'corenlp.CoreNLPAnalyzer'), o('annotators', *'tokenize ssplit pos lemma ner'.split)),
    'fullcorenlp' => l(o('LanguageAnalyzer', 'corenlp.CoreNLPAnalyzer'), o('annotators', *'tokenize ssplit pos lemma ner parse'.split)),
  }),
  # Training
  letDefault(:train, 0),
  sel(:train,
    l(),
    l(
      o('combineFromFloatingParser'),
      sel(:unseen, unbalancedTrainDevSplit, l()),
      o('maxTrainIters', 3),
      o('showValues', false), o('showFirstValue'),
    nil),
    l(
      # for dumping derivations (@class=dump)
      # force unbalancedTrainDevSplit + combine from floating parser
      o('combineFromFloatingParser'), o('DPParser.cheat'),
      sel(:unseen, unbalancedTrainDevSplit, l()),
    nil),
  nil),
  # Regularization
  letDefault(:l1, 1),
  sel(:l1,
    l(),
    l(o('Params.l1Reg','lazy'), o('Params.l1RegCoeff', '3e-5')),
    l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0, 0.00001, 0.0001, 0.001, 0.01)),
    l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0.00001, 0.00003, 0.0001, 0.0003)),
    l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0.00001, 0.00003, 0.0005)),
  nil),
  # Features
  letDefault(:feat, 'none'),
  sel(:feat, {
    'none' => l(),   # No features (random)
    'some' => l(     # Add your own features! (only set up the feature computers)
      o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
    nil),
    'all' => l(      # All features
      o('FeatureExtractor.featureDomains', 'custom-denotation phrase-predicate phrase-denotation headword-denotation missing-predicate'.split),
      o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
    nil),
    'baseline' => l( # For the baseline classifier
      o('FeatureExtractor.featureDomains', 'custom-denotation phrase-denotation headword-denotation table-baseline'.split),
      o('FeatureExtractor.featureComputers', 'tables.baseline.TableBaselineFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
    nil),
    'ablate' => l(
      o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
      selo(nil,
        'FeatureExtractor.featureDomains',
        'phrase-predicate phrase-denotation headword-denotation missing-predicate'.split,
        'custom-denotation phrase-denotation headword-denotation missing-predicate'.split,
        'custom-denotation phrase-predicate headword-denotation missing-predicate'.split,
        'custom-denotation phrase-predicate phrase-denotation missing-predicate'.split,
        'custom-denotation phrase-predicate phrase-denotation headword-denotation'.split,
      nil),
    nil),
  }),
  letDefault(:featOp, 'careful'),
  sel(:featOp, {
    'none' => l(),
    'base' => l(
      o('usePredicateLemma'), o('usePhraseLemmaOnly'),
    nil),
    'careful' => l(
      o('usePredicateLemma'), o('usePhraseLemmaOnly'),
      o('maxNforLexicalizeAllPairs', 2),
      o('traverseWithFormulaTypes'), o('reverseNameValueConversion', 'allBang'),
      o('lookUnderCellProperty'), o('useGenericCellType'),
      o('computeFuzzyMatchPredicates'),
    nil),
  }),
nil) })

def tablesGrammarPaths
  lambda { |e|
    baseDir = ['tables/grammars/', 'grammars/'][e[:cldir]]
    l(
      letDefault(:grammar, 'combined-all'),
      sel(:grammar, {
        'restrict' => o('Grammar.inPaths', "#{baseDir}restrict.grammar"),
        'simple' => o('Grammar.inPaths', "#{baseDir}simple.grammar"),
        'combined' => o('Grammar.inPaths', "#{baseDir}combined.grammar"),
        'combined-jnc' => l(    # WQ baseline
          o('Grammar.inPaths', "#{baseDir}combined.grammar"),
          o('Grammar.tags', *'movement count'.split),
        nil),
        'combined-cut' => l(    # No intersection / union
          o('Grammar.inPaths', "#{baseDir}combined.grammar"),
          o('Grammar.tags', *'movement comparison count aggregate superlative arithmetic'.split),
        nil),
        'combined-all' => l(    # Default
          o('Grammar.inPaths', "#{baseDir}combined.grammar"),
          o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge'.split),
        nil),
        'combined-trigger' => l(    # Use trigger words for operations
          o('Grammar.inPaths', "#{baseDir}combined.grammar"),
          o('Grammar.tags', *'t-alternative t-movement t-comparison t-count t-aggregate t-superlative t-arithmetic merge'.split),
        nil),
      }),
    nil)
  }
end

def tablesDataPaths
  lambda { |e|
    baseDir = ['lib/data/tables/data/', 'WikiTableQuestions/data/'][e[:cldir]]
    csvDir = ['lib/data/tables/', 'WikiTableQuestions/'][e[:cldir]]
    datasets = {
      'none' => l(),
      # Pristine test test
      'test' => l(
        o('Dataset.inPaths',
          "train,#{baseDir}training.examples",
          "test,#{baseDir}pristine-unseen-tables.examples"),
        o('Dataset.trainFrac', 1), o('Dataset.devFrac', 0),
        let(:unseen, 1),
      nil),
    }
    # Development sets: 80:20 random split of training data
    ['1', '2', '3', '4', '5'].each do |x|
      datasets['u-' + x] = l(
        o('Dataset.inPaths',
          "train,#{baseDir}random-split-seed-#{x}-train.examples",
          "dev,#{baseDir}random-split-seed-#{x}-test.examples",
        nil),
        let(:unseen, 1),
      nil)
    end
    # That's it!
    l(
      o('TableKnowledgeGraph.baseCSVDir', csvDir),
      sel(:data, datasets),
    nil)
  }
end

def tablesPruningStrategies
  [
    # Formula
    "singleton",
    "multipleSuperlatives",
    "sameMerge",
    "forwardBackward",
    "doubleNext",
    # Denotation
    "emptyDenotation",
    "nonLambdaError",
    "tooManyValues",
    "badSuperlativeHead",
    "mistypedMerge",
  nil].compact
end



# {2015-01-18} Generate utterances [Percy]
addMode('genovernight', 'Generate utterances for overnight semantic parsing', lambda { |e| l(
  'fig/bin/qcreate',
  letDefault(:gen, 0),
  sel(:gen, l()),
  'java', '-Dmodules=core,overnight', '-Xmx10g', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.overnight.GenerationMain',
  figOpts,
  o('JoinFn.typeInference', true),
  o('JoinFn.specializedTypeCheck', false),

  o('JavaExecutor.convertNumberValues', false),
  o('JavaExecutor.printStackTrace', false),

  # These domains are all based on SimpleWorld
  required(:domain),
  o('Grammar.inPaths', lambda { |e| 'overnight/' + e[:domain] + '.grammar' }),
  o('SimpleWorld.domain', lambda { |e| e[:domain] }),
  o('initialization', 'denotation :: error,-1000', 'denotation :: empty,-100', 'paraphrase :: size,+0.01', 'denotation :: value_in_formula,-100'),
  o('FeatureExtractor.featureComputers','overnight.OvernightFeatureComputer'),
  o('OvernightFeatureComputer.featureDomains', ''),
  o('OvernightFeatureComputer.itemAnalysis',false),
  sel(:gen,
    l( # For debugging the grammar
      o('FeatureExtractor.featureDomains', 'denotation'),
      o('Dataset.inPaths', lambda { |e| 'train:overnight/' + e[:domain] + '-unittest.examples'}),
      selo(:parse, 'Grammar.tags', 'generate', 'parse'),
      o('interactive'),
    nil),
    l( # For generating utterances
      o('parser', 'FloatingParser'),
      o('maxDepth', 30), o('beamSize', 10000),
      o('derivationScoreNoise', 1),
      o('Dataset.inPaths', 'train:overnight/null.examples'),
      o('Derivation.showUtterance'),
      o('FeatureExtractor.featureDomains', 'denotation'),
      o('printAllPredictions'),
      o('printPredictedUtterances'),
      o('executeAllDerivations'),
      o('Parser.pruneErrorValues', true),
      o('Grammar.tags', 'generate'),
    nil),
  nil),
  a('Grammar.tags', 'general'),
nil) })

# Generate for all the domains
addMode('genovernight-wrapper', 'Generate utterances for overnight semantic parsing', lambda { |e| l(
  './run', '@mode=genovernight', '@gen=1',
  sel(nil, {
    'calendar' => let(:domain, 'calendar'),
    'blocks' => let(:domain, 'blocks'),
    'housing' => let(:domain, 'housing'),
    'restaurants' => let(:domain, 'restaurants'),
    'publications' => let(:domain, 'publications'),
    'socialnetwork' => let(:domain, 'socialnetwork'),
    'basketball' => let(:domain, 'basketball'),
    'geo880' => let(:domain, 'geo880'),
    'recipes' => let(:domain, 'recipes'),
  }),
  lambda { |e| '@domain=' + e[:domain] },
  lambda { |e| system 'mkdir -p genovernight.out'; o('execDir', 'genovernight.out/' + e[:domain]) },
nil) })

addMode('geo880', 'Semantic parsing on the geo880 dataset', lambda { |e| l(
  # Usual header
  header('core,tables,corenlp,geo880'),
  'edu.stanford.nlp.sempre.Main',
  # Fig parameters
  figOpts,
  o('executor', 'tables.lambdadcs.LambdaDCSExecutor'),
  o('JoinFn.specializedTypeCheck', false), o('JoinFn.typeInference', false),
  # Parser
  o('Builder.parser', 'BeamParser'),
  o('Parser.coarsePrune'),
 
  # Evaluation
  o('Builder.valueEvaluator', 'geo880.Geo880ValueEvaluator'),

  # Grammar
  o('Grammar.inPaths','lib/data/geo880/geo880.grammar'),

  # Type hierarchy
  o('Geo880TypeLookup.typeHierarchyPath', 'lib/data/geo880/geo880.type_hierarchy'),
  o('TypeInference.typeLookup','geo880.Geo880TypeLookup'),

  # Yrkvpba
  o('SimpleLexicon.inPaths', 'lib/data/geo880/geo880.lexicon'),
  
  # Learner
  o('Learner.maxTrainIters', 3),

  # Dataset
  letDefault(:data, 0),
  sel(:data,
    l(o('Dataset.inPaths', 'train,lib/data/geo880/geo880-train.preprocessed.examples'), unbalancedTrainDevSplit), # (0) train 0.8, dev 0.2
    l(o('Dataset.inPaths', 'train,lib/data/geo880/geo880-train.examples', 'test,lib/data/geo880/geo880-test.preprocessed/examples')), # (1) Don't run on test yet!
  nil),
  # Load the graph
  o('Dataset.globalGraphPath', 'lib/data/geo880/geo880.kg'),
  # Verbosity
  letDefault(:verbose, 0),
  sel(:verbose,
    l(),
    l(
      o('showRules'),
      o('Parser.verbose', 2),
      o('JoinFn.verbose', 3),
      o('JoinFn.showTypeCheckFailures'),
    nil),
  nil),
  # Language Analyzer
  l(o('LanguageAnalyzer', 'corenlp.CoreNLPAnalyzer'), o('annotators', *'tokenize ssplit pos lemma ner'.split)),
  # Regularization
  letDefault(:l1, 0),
  sel(:l1,
    l(),
    l(o('Params.l1Reg','lazy'), o('Params.l1RegCoeff', '3e-5')),
    l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0, 0.00001, 0.0001, 0.001, 0.01)),
  nil),
  # Features
  letDefault(:feat, 'freebase'),
  sel(:feat, {
    'none' => l(),   # No features (random)
    'freebase' => l(
      o('FeatureExtractor.featureDomains', 'rule opCount constant whType span lemmaAndBinaries denotation lexAlign joinPos skipPos'.split),
#      o('FeatureExtractor.featureDomains', 'rule opCount constant whType lemmaAndBinaries denotation lexAlign joinPos skipPos'.split),
    nil),
  }),
nil) })

############################################################

if ARGV.size == 0
  puts "#{$0} @mode=<mode> [options]"
  puts
  puts 'This is the main entry point for all SEMPRE-related runs.'
  puts "Modes:"
  $modes.each { |name,description,func|
    puts "  #{name}: #{description}"
  }
end

modesMap = {}
$modes.each { |name,description,func|
  modesMap[name] = func
}
run!(sel(:mode, modesMap))