-
Notifications
You must be signed in to change notification settings - Fork 33
/
Copy pathconfig-full.toml
245 lines (199 loc) · 10.4 KB
/
config-full.toml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
# Example complete configuration for DVID with multiple database backends assigned
# per data type and data instance.
[server]
host = "mygreatserver.test.com" # Lets you specify a user-friendly alias for help messages.
httpAddress = ":8000"
rpcAddress = ":8001"
webClient = "/path/to/webclient"
# only one of the following should be used if the default file path is missing, with webRedirectPath
# taking priority if both are specified.
webDefaultFile = "index.html" # contents of this file will be returned for any bad file path; required by dvid-console 3.0+
webRedirectPath = "/index.html" # if supplied, bad file paths will cause redirection to this path
# if specified, dvid will write its process ID to the file on server start, and remove it on closing.
# pidFile = "/path/to/dvid-pidfile"
note = """
You can put anything you want in here and have it available via /api/server/note.
Multiple lines!
"""
interactiveOpsBeforeBlock = 10 # Blocks processing routines, e.g., image tile creations, if interactive ops over last 2 min exceeds this amount. If omitted or 0 will do no blocking.
shutdownDelay = 0 # Delay after shutdown request to let HTTP requests drain. Default is 5 seconds.
corsDomains = ["mydomain.org"] # Allow CORS origin in these domains (not subdomains)
# If ["*"] will use * wildcard for origins.
# If empty array or omitted, no Access-Control-Allow-Origin will be set.
noLabelmapSplit = true # Default false. If true, won't allow /split endpoint on labelmap.
# Blocking that can will be read on server startup or calling POST /api/server/reload-blocklist.
# Each line should be one of the following:
# u=someuserid,optional note to be sent with the 429 (Too Many Requests) response code
# ip=23.10.5.*,optional note to be sent with the 429 (Too Many Requests) response code
blockListFile = "../scripts/distro-files/blocklist-example.csv"
# rwmode settings allow read-only and full write modes similar to command-line flags
# rwmode = "readonly"
# rwmode = "fullwrite"
# if a start-up webhook is provided, DVID will do a POST on the webhook address and send JSON
# with the server attributes including the values for "host", "note", and other server properties.
# startWebhook = "http://dvidmonitor.hhmi.org"
# to return Timing-Allow-Origin headers in response
# allowTiming = true
# How new data instance ids are generated.
# Is one of "random" or "sequential". If "sequential" can set "start_instance_id" property.
# Use of "random" is a cheap way to have multiple frontend DVIDs use a shared store without
# key collisions.
instance_id_gen = "sequential"
instance_id_start = 100 # new ids start at least from this.
min_mutation_id_start = 1000100000 # mutation id will start from this or higher
# if the "auth" section is included, requests must be signed with a JWT obtained via
# the /api/server/token endpoint. There are two methods of authentication:
# The current approach is that a Google ID token is passed to /api/server/token and
# a JWT is returned that should be sharable among other FlyEM services. The environment
# variable DVID_JWT_SECRET_KEY must be provided for authentication to be enabled.
# The legacy approach is to supply a deprecated proxy_address to a flyem-services server.
[auth]
enforce = "none" # "none" = no JWT required, "token" requires valid JWT, and
# "authfile" requires user to be in auth_file below.
# File containing authorized emails in JSON format of style {"email":"priv", ...}
# where email could be "*" for wildcard and priv can be "read", "write", or "readwrite".
#auth_file = "/demo/auth.txt"
public_versions = [] # Add lists of committed UUIDs that should have public access.
# This is legacy method that is deprecated and won't be used.
# proxy_address = "http://some-flyem-services" # URL to running github.com/janelia-flyem/flyem-services
# Email server to use for notifications and server issuing email-based authorization tokens.
[email]
notify = ["foo@someplace.edu"] # Who to send email in case of panic
username = "myuserid"
password = "mypassword"
server = "mail.myserver.com"
port = 25
[logging]
logfile = "/demo/logs/dvid.log"
max_log_size = 500 # MB
max_log_age = 30 # days
[mutations]
# store JSON mutation records for selected instances in Data+Version UUID log files.
jsonstore = "/path/to/json-mutlog"
# use kafka server with "my-mutations" topic.
# httpstore = "kafka:my-mutations"
# use local log store with automatic UUID topic
httpstore = "logstore:mutationlog"
# store any large POST body into blobstore with unique ref stored in kafka
blobstore = "raid6"
# Backends can be specified in many ways. In decreasing order of precedence:
#
# backend."<name>:<uuid>" = store to use for a particular data instance,
# where uuid is the full UUID of the data instance's root in the DAG.
# backend."<tag key>:<tag value>" = store to use for particular data instance
# tags. See "type" tag with value "meshes" below.
# backend.<datatype> = store to use for the given "datatype"
# backend.metadata = store to use for metadata
# backend.default = default storage engine if not otherwise specified
#
# If no backend is specified, DVID will return an error unless there is only
# one store, which will automatically be backend.default.
[backend]
[backend.default]
store = "raid6"
log = "mutationlog"
[backend.labelblk] # This maps all instances of labelblk type
store = "ssd"
[backend.labelmap] # This maps all instances of labelmap type
store = "ssd"
log = "labelmutationlog"
[backend."2ff3197a5a9844be9a905b38d12d41ca"] # if only UUID, it's instance Data UUID
store = "badger"
[backend."grayscale:99ef22cd85f143f58a623bd22aad0ef7"] # instance name : version UUID
store = "badger"
[backend."grayscale2:83698c997b214365abd479cfc7744049"]
store = "ng"
[backend."type:meshes"]
store = "raid6"
# List the different storage systems available for metadata, data instances, etc.
# Any nickname can be used for a backend. In this case, it's "raid6" to reflect
# that the directory is on a RAID-6 drive system, "ssd" for a directory mounted on
# a SSD, and "kvautobus" for an internal Janelia HTTP dataservice. Note that all
# store properties like "engine" and "path" should be lower-case by convention.
[store]
[store.raid6]
engine = "badger"
path = "/data/dbs/badgerdb"
[store.ssd]
engine = "badger"
path = "/datassd/dbs/badgerdb"
[store.badger]
engine = "badger"
path = "/path/to/badger"
[store.mutationlog]
engine = "filelog"
path = "/data/mutationlog" # directory that holds mutation log per instance-UUID.
[store.labelmutationlog]
engine = "filelog"
path = "/data/labelmutlog" # directory that holds mutation log per instance-UUID.
[store.neuronjson]
engine = "badger"
path = "/path/to/badger"
inmemory = ["83698c997b214365abd479cfc7744049", ":mybranch"] # in-memory DBs in addition to main HEAD.
# The above can contain committed UUIDs (static) or ":branch" that always resolve to branch HEAD.
# By default, the HEAD of the master branch is always in memory.
[store.ng]
engine = "ngprecomputed"
ref = "the-GCS-bucket-name" # note this is ref and not path
instance = "basename" # if supplied, auto creates multi-scale uint8blk instances
# Kafka support can be specified. This allows mutations to be logged and facilitates
# syncing, etc. If a "filelog" store is available as default, then any failed kafka
# messages will be stored in a file named for the topic.
[kafka]
# optional: forces topic name for activity.
topicActivity = "allMyActivity"
# optional: adds prefix to any mutation logging
topicPrefix = "postsFromServer1"
# optional: forces topic suffix for instance mutations; each entry is data UUID : suffix.
topicSuffixes = ["bc95398cb3ae40fcab2529c7bca1ad0d:myGreatDataInstance"]
# optional: allows setting of max # messages on producer queue (queue.buffering.max.messages)
# default is 100,000 and may be exceeded for activity logs on very busy servers
bufferSize = 1000000
# kafka config "bootstrap.servers"
servers = ["foo.bar.com:1234", "foo2.bar.com:1234"]
# kafka config "security.protocol"
secProtocol = "SASL_SSL"
# kafka config "sasl.mechanisms"
saslMechanisms = "PLAIN"
# kafka config "sasl.username"
saslUsername = "KEY-TAKEN-FROM-CONFLUENT-CLOUD"
# kafka config "sasl.password"
saslPassword = "SECRET-TAKEN-FROM-CONFLUENT-CLOUD"
# Mutation cache, currently supported only for labelmap's label indices,
# stores every label index just before modification. This allows you to
# reach back to its state just before any given mutation id.
# For each labelmap instance, the instance name should specify its
# path to a database cache.
[mutcache]
[mutcache.segmentation]
path = "/path/to/database/cache/for/instance/segmentation"
# Cache support allows setting datatype-specific caching mechanisms.
# Currently freecache is supported in labelarray and labelmap.
# This experimental feature allows caching of label indices.
[cache]
[cache.labelmap]
size = 1000 # MB
# Groupcache support lets you cache GETs from particular data instances using a
# distributed, immutable key-value cache.
#
# The configuration below marks some data instances as both immutable and
# using a non-ordered key-value store for GETs. These instances may be versioned.
# An example case would be imagetile, which is immutable after some initial set
# of ingestions that may be spread across a few versions at the top of the DAG.
[groupcache]
gb = 60 # 60 GB if we have a beefy server
host = "http://10.0.0.1:8003"
peers = ["http://10.0.0.2:8002", "http://10.0.0.3:8002"] # currently not used
instances = ["graytiles:99ef22cd85f143f58a623bd22aad0ef7"]
# Mirror support sets up echoing of POST requests to remote dvid servers.
# Mirroring can be limited to specified instances under the caveat that this
# can cause remote identical UUIDs to have partially mutated data instead of fully
# mirrored data. This can be useful for load splitting where batch processes can
# hit the mirror for the given instances.
[mirror]
# specify mirror for all POSTs across instances
[mirror.all]
servers = ["http://mirror1.janelia.org:7000", "http://mirror2.janelia.org:7000"]
# specify mirror for this data UUID and particular version UUID
[mirror."bc95398cb3ae40fcab2529c7bca1ad0d:99ef22cd85f143f58a623bd22aad0ef7"]
servers = ["http://mirror3.janelia.org:7000", "http://mirror4.janelia.org:7000"]