mrjob.conf

runners:
  emr:
    applications:
    - Spark
    - Livy
    bootstrap:
    - sudo python3 -m pip install --upgrade pip
    - sudo yum -y install gcc postgresql-devel unzip
    - sudo python3 -m pip install -r requirements.txt#
    - sudo unzip -d /home/hadoop/quant quant.zip#
    cmdenv:
      PYSPARK_PYTHON: /usr/bin/python3
      PYTHONPATH: /home/hadoop/quant:$PYTHONPATH
    max_mins_idle: 20
    num_core_instances: 8
    instance_type: c5.2xlarge
    pool_clusters: True
    region: us-west-1
    release_label: emr-5.23.0
    emr_configurations:
    - Classification: spark-defaults
      Properties:
        spark.executor.memory: 4g
        spark.driver.memory: 4g
        spark.driver.memoryOverhead: 2g