-
Notifications
You must be signed in to change notification settings - Fork 606
/
Copy pathlivy.conf.template
261 lines (219 loc) · 12 KB
/
livy.conf.template
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Use this keystore for the SSL certificate and key.
# livy.keystore =
# Specify the keystore password.
# livy.keystore.password =
#
# Specify the key password.
# livy.key-password =
# Name of the keystore implementation that was used when generating the keystore
# livy.keystore.type = JKS
# Hadoop Credential Provider Path to get "livy.keystore.password" and "livy.key-password".
# Credential Provider can be created using command as follow:
# hadoop credential create "livy.keystore.password" -value "secret" -provider jceks://hdfs/path/to/livy.jceks
# livy.hadoop.security.credential.provider.path =
# What host address to start the server on. By default, Livy will bind to all network interfaces.
# livy.server.host = 0.0.0.0
# What port to start the server on.
# livy.server.port = 8998
# What base path ui should work on. By default UI is mounted on "/".
# E.g.: livy.ui.basePath = /my_livy - result in mounting UI on /my_livy/
# livy.ui.basePath = ""
# What spark master Livy sessions should use.
# livy.spark.master = local
# What spark deploy mode Livy sessions should use.
# livy.spark.deploy-mode =
# Configure Livy server http request and response header size.
# livy.server.request-header.size = 131072
# livy.server.response-header.size = 131072
# Whether or not to send server version in http response.
# livy.server.send-server-version = false
# Enabled to check whether timeout Livy sessions should be stopped.
# livy.server.session.timeout-check = true
#
# Whether or not to skip timeout check for a busy session
# livy.server.session.timeout-check.skip-busy = false
# Time in milliseconds on how long Livy will wait before timing out an inactive session.
# Note that the inactive session could be busy running jobs.
# livy.server.session.timeout = 1h
#
# How long a finished session state should be kept in LivyServer for query.
# livy.server.session.state-retain.sec = 600s
# If livy should impersonate the requesting users when creating a new session.
# livy.impersonation.enabled = false
# Logs size livy can cache for each session/batch. 0 means don't cache the logs.
# livy.cache-log.size = 200
# Comma-separated list of Livy RSC jars. By default Livy will upload jars from its installation
# directory every time a session is started. By caching these files in HDFS, for example, startup
# time of sessions on YARN can be reduced.
# livy.rsc.jars =
# Comma-separated list of Livy REPL jars. By default Livy will upload jars from its installation
# directory every time a session is started. By caching these files in HDFS, for example, startup
# time of sessions on YARN can be reduced. Please list all the repl dependencies including
# Scala version-specific livy-repl jars, Livy will automatically pick the right dependencies
# during session creation.
# livy.repl.jars =
# Location of PySpark archives. By default Livy will upload the file from SPARK_HOME, but
# by caching the file in HDFS, startup time of PySpark sessions on YARN can be reduced.
# livy.pyspark.archives =
# Location of the SparkR package. By default Livy will upload the file from SPARK_HOME, but
# by caching the file in HDFS, startup time of R sessions on YARN can be reduced.
# livy.sparkr.package =
# List of local directories from where files are allowed to be added to user sessions. By
# default it's empty, meaning users can only reference remote URIs when starting their
# sessions.
# livy.file.local-dir-whitelist =
# Whether to enable csrf protection, by default it is false. If it is enabled, client should add
# http-header "X-Requested-By" in request if the http method is POST/DELETE/PUT/PATCH.
# livy.server.csrf-protection.enabled =
# Whether to enable HiveContext in livy interpreter, if it is true hive-site.xml will be detected
# on user request and then livy server classpath automatically.
# livy.repl.enable-hive-context =
# Recovery mode of Livy. Possible values:
# off: Default. Turn off recovery. Every time Livy shuts down, it stops and forgets all sessions.
# recovery: Livy persists session info to the state store. When Livy restarts, it recovers
# previous sessions from the state store.
# Must set livy.server.recovery.state-store and livy.server.recovery.state-store.url to
# configure the state store.
# livy.server.recovery.mode = off
# Zookeeper address used for HA and state store. e.g. host1:port1, host2:port2
# livy.server.zookeeper.url =
# Where Livy should store state to for recovery. Possible values:
# <empty>: Default. State store disabled.
# filesystem: Store state on a file system.
# zookeeper: Store state in a Zookeeper instance.
# livy.server.recovery.state-store =
# For filesystem state store, the path of the state store directory. Please don't use a filesystem
# that doesn't support atomic rename (e.g. S3). e.g. file:///tmp/livy or hdfs:///.
# For zookeeper, the address to the Zookeeper servers. e.g. host1:port1,host2:port2
# If livy.server.recovery.state-store is zookeeper, this config is for back-compatibility,
# so if both this config and livy.server.zookeeper.url exist,
# livy uses livy.server.zookeeper.url first.
# livy.server.recovery.state-store.url =
# The policy of curator connecting to zookeeper.
# For example, m, n means retry m times and the interval of retry is n milliseconds.
# Please use the new config: livy.server.zk.retry-policy.
# Keep this config for back-compatibility.
# If both this config and livy.server.zk.retry-policy exist,
# livy uses livy.server.zk.retry-policy first.
# livy.server.recovery.zk-state-store.retry-policy = 5,100
# The policy of curator connecting to zookeeper.
# For example, m, n means retry m times and the interval of retry is n milliseconds
# livy.server.zk.retry-policy =
# The dir in zk to store the data about session.
# livy.server.recovery.zk-state-store.key-prefix = livy
# If Livy can't find the yarn app within this time, consider it lost.
# livy.server.yarn.app-lookup-timeout = 120s
# When the cluster is busy, we may fail to launch yarn app in app-lookup-timeout, then it would
# cause session leakage, so we need to check session leakage.
# How long to check livy session leakage
# livy.server.yarn.app-leakage.check-timeout = 600s
# how often to check livy session leakage
# livy.server.yarn.app-leakage.check-interval = 60s
# How often Livy polls YARN to refresh YARN app state.
# livy.server.yarn.poll-interval = 5s
#
# Days to keep Livy server request logs.
# livy.server.request-log-retain.days = 5
# If the Livy Web UI should be included in the Livy Server. Enabled by default.
# livy.ui.enabled = true
# Whether to enable Livy server access control, if it is true then all the income requests will
# be checked if the requested user has permission.
# livy.server.access-control.enabled = false
# Allowed users to access Livy, by default any user is allowed to access Livy. If user want to
# limit who could access Livy, user should list all the permitted users with comma separated.
# livy.server.access-control.allowed-users = *
# A list of users with comma separated has the permission to change other user's submitted
# session, like submitting statements, deleting session.
# livy.server.access-control.modify-users =
# A list of users with comma separated has the permission to view other user's infomation, like
# submitted session state, statement results.
# livy.server.access-control.view-users =
#
# Authentication support for Livy server
# Livy has a built-in SPnego authentication support for HTTP requests with below configurations.
# livy.server.auth.type = kerberos
# livy.server.auth.kerberos.principal = <spnego principal>
# livy.server.auth.kerberos.keytab = <spnego keytab>
# livy.server.auth.kerberos.name-rules = DEFAULT
#
# If user wants to use custom authentication filter, configurations are:
# livy.server.auth.type = <custom>
# livy.server.auth.<custom>.class = <class of custom auth filter>
# livy.server.auth.<custom>.param.<foo1> = <bar1>
# livy.server.auth.<custom>.param.<foo2> = <bar2>
# Enable to allow custom classpath by proxy user in cluster mode
# The below configuration parameter is disabled by default.
# livy.server.session.allow-custom-classpath = true
# value specifies interval to check safe mode in hdfs filesystem
# livy.server.hdfs.safe-mode.interval = 5
# value specifies max attempts to retry when safe mode is ON in hdfs filesystem
# livy.server.hdfs.safe-mode.max.retry.attempts = 10
# Manual authentication to KubeApiserver (by default configured with Kubernetes ServiceAccount
# if deployed to Kubernetes cluster as a Pod)
# Kubernetes oauth token file path
# livy.server.kubernetes.oauthTokenFile =
# Kubernetes oauth token string value
# livy.server.kubernetes.oauthTokenValue =
# Kubernetes CA cert file path
# livy.server.kubernetes.caCertFile =
# Kubernetes client key file path
# livy.server.kubernetes.clientKeyFile =
# Kubernetes client cert file path
# livy.server.kubernetes.clientCertFile =
# If Livy can't find the Kubernetes app within this time, consider it lost.
# livy.server.kubernetes.app-lookup-timeout = 600s
# If Livy can't find the Kubernetes app within this max times, consider it lost.
# livy.server.kubernetes.app-lookup.max-failed.times = 120
# The size of thread pool to monitor all Kubernetes apps.
# livy.server.kubernetes.app-lookup.thread-pool.size = 4
# When the cluster is busy, we may fail to launch yarn app in app-lookup-timeout, then it would
# cause session leakage, so we need to check session leakage.
# How long to check livy session leakage
# livy.server.kubernetes.app-leakage.check-timeout = 600s
# How often to check livy session leakage
# livy.server.kubernetes.app-leakage.check-interval = 60s
# How often Livy polls KubeApiServer to refresh KubernetesApp state (Pods state, logs, description
# details, routes, etc...)
# livy.server.kubernetes.poll-interval = 15s
# Weather to create Kubernetes Nginx Ingress for Spark UI. If set to true, configure the desired
# options below
# livy.server.kubernetes.ingress.create = false
# Kubernetes Nginx Ingress protocol. If set to https refer Ingress TLS section below
# livy.server.kubernetes.ingress.protocol = http
# Kubernetes Nginx Ingress host. Be sure to set it to the FQDN of your Nginx Ingress Controller
# proxy server
# livy.server.kubernetes.ingress.host = localhost
# Kubernetes secret name for Nginx Ingress TLS. Is omitted if 'livy.server.kubernetes.ingress.protocol'
# is not https
# livy.server.kubernetes.ingress.tls.secretName = spark-cluster-tls
# Kubernetes Nginx Ingress additional configuration snippet for specific config options
# livy.server.kubernetes.ingress.additionalConfSnippet =
# Kubernetes Nginx Ingress additional annotations for specific config options, eg. for configuring
# basic auth of external oauth2 proxy. Format: annotation1=value1;annotation2=value2;...
# livy.server.kubernetes.ingress.additionalAnnotations =
# Set to true to enable Grafana Loki integration and configure options below
livy.server.kubernetes.grafana.loki.enabled = false
# Grafana UI root endpoint to build links based on
# livy.server.kubernetes.grafana.url = http://localhost:3000
# Grafana Datasource name for Loki
# livy.server.kubernetes.grafana.loki.datasource = loki
# Time range from now to past to get logs for
# livy.server.kubernetes.grafana.timeRange = 6h
# Used to build links to Spark History Server pages on Spark App completion (Kubernetes only)
# livy.ui.history-server-url = http://spark-history-server