forked from flyteorg/flytepropeller
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathconfig.yaml
99 lines (99 loc) · 2.79 KB
/
config.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
# This is a sample configuration file.
# Real configuration when running inside K8s (local or otherwise) lives in a ConfigMap
propeller:
workers: 4
workflow-reeval-duration: 10s
downstream-eval-duration: 5s
limit-namespace: "all"
prof-port: 11254
metrics-prefix: flyte
enable-admin-launcher: true
max-ttl-hours: 1
gc-interval: 500m
queue:
type: batch
queue:
type: bucket
rate: 20
capacity: 100
sub-queue:
type: bucket
rate: 100
capacity: 1000
kube-config: "$HOME/.kube/config"
publish-k8s-events: true
# Sample plugins config
plugins:
# Set of enabled plugins at root level
enabled-plugins:
- container
- waitable
- K8S-ARRAY
# All k8s plugins default configuration
k8s:
inject-finalizer: true
default-annotations:
- annotationKey1: annotationValue1
resource-tolerations:
nvidia.com/gpu:
key: flyte/gpu
value: dedicated
operator: Equal
effect: NoSchedule
default-env-vars:
- AWS_METADATA_SERVICE_TIMEOUT: 5
- AWS_METADATA_SERVICE_NUM_ATTEMPTS: 20
- FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000"
- FLYTE_AWS_ACCESS_KEY_ID: minio
- FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage
# Spark Plugin configuration
spark:
spark-config-default:
- spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2"
- spark.kubernetes.allocation.batch.size: "50"
- spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl"
- spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem"
- spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A"
- spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem"
- spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A"
- spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem"
- spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A"
- spark.hadoop.fs.s3a.multipart.threshold: "536870912"
- spark.blacklist.enabled: "true"
- spark.blacklist.timeout: "5m"
# Waitable plugin configuration
waitable:
console-uri: http://localhost:30081/console
# Logging configuration
logs:
kubernetes-enabled: true
kubernetes-url: "http://localhost:30082"
storage:
connection:
access-key: minio
auth-type: accesskey
disable-ssl: true
endpoint: http://localhost:9000
region: us-east-1
secret-key: miniostorage
cache:
max_size_mbs: 10
target_gc_percent: 100
container: myflytecontainer
type: minio
event:
type: admin
rate: 500
capacity: 1000
admin:
endpoint: localhost:8089
insecure: true
catalog-cache:
type: catalog
endpoint: datacatalog:8089
insecure: true
errors:
show-source: true
logger:
level: 4
show-source: true