Skip to content

Commit

Permalink
Merge pull request #130 from WorldCereal/129-custom-joboptions
Browse files Browse the repository at this point in the history
#129 Add custom job_options
  • Loading branch information
GriffinBabe authored Aug 23, 2024
2 parents 788725e + 3a634c0 commit 8e91168
Showing 1 changed file with 15 additions and 6 deletions.
21 changes: 15 additions & 6 deletions src/worldcereal/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ def generate_map(
out_format: str = "GTiff",
backend_context: BackendContext = BackendContext(Backend.FED),
tile_size: Optional[int] = 128,
job_options: Optional[dict] = None,
) -> InferenceResults:
"""Main function to generate a WorldCereal product.
Expand All @@ -209,6 +210,8 @@ def generate_map(
backend to run the job on
tile_size: int, optional
Tile size to use for the data loading in OpenEO, by default 128.
job_options: dict, optional
Additional job options to pass to the OpenEO backend, by default None
Returns
-------
Expand Down Expand Up @@ -267,15 +270,21 @@ def generate_map(
)

# Submit the job
JOB_OPTIONS = {
"driver-memory": "4g",
"executor-memory": "1g",
"executor-memoryOverhead": "1g",
"python-memory": "2g",
"soft-errors": "true",
"udf-dependency-archives": [f"{ONNX_DEPS_URL}#onnx_deps"],
}
if job_options is not None:
JOB_OPTIONS.update(job_options)

job = classes.execute_batch(
outputfile=output_path,
out_format=out_format,
job_options={
"driver-memory": "4g",
"executor-memory": "3g",
"executor-memoryOverhead": "5g",
"udf-dependency-archives": [f"{ONNX_DEPS_URL}#onnx_deps"],
},
job_options=JOB_OPTIONS,
)

asset = job.get_results().get_assets()[0]
Expand Down

0 comments on commit 8e91168

Please sign in to comment.