-
Notifications
You must be signed in to change notification settings - Fork 30
/
Copy pathmain.yml
121 lines (99 loc) · 4.99 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
# Post-deployment options
dw_install_web_client: True
dw_bounce_accumulo: True
dw_ingest_tvmaze: False
dw_ingest_wikipedia: False
dw_start_ingest: True
dw_start_web: False
# Force rebuild/redeploy
dw_force_redeploy: False
# Build
dw_build_profile: "dev"
dw_project_version: "2.6.41"
dw_clone_dir: "{{ user_home }}/datawave-source"
dw_repo: "https://github.com/NationalSecurityAgency/datawave.git"
dw_checkout_version: "2.6.41"
dw_build_property_overrides_dir: "{{ user_home }}/.m2/datawave/properties"
dw_m2_repo_dir: "{{ user_home }}/.m2/repository"
dw_build_command: "mvn -P{{ dw_build_profile }} -Dmaven.repo.local={{ dw_m2_repo_dir }} -Ddeploy -Dtar -Ddist -DskipTests clean install"
# Accumulo
dw_accumulo_root_auths: [PUBLIC,PRIVATE,FOO,BAR,DEF,A,B,C,D,E,F,G,H,I,DW_USER,DW_SERV,DW_ADMIN,JBOSS_ADMIN]
dw_accumulo_vfs_context_name: "datawave"
dw_accumulo_vfs_context_dir: "/{{ dw_accumulo_vfs_context_name }}/accumulo-classpath"
dw_accumulo_vfs_context_config: "{{ hdfs_root }}{{ dw_accumulo_vfs_context_dir }}/.*.jar"
# By default, i.e., when 'dw_accumulo_skip_shell_tasks == False', the Accumulo shell commands
# below will run each time Accumulo is bounced via the restart-accumulo.yml playbook. So,
# make sure there are no commands included that will be problematic to have executed multiple
# times. Note that Accumulo 'TableExistsException' is ignored by the shell task handler.
# Otherwise, ensure that 'dw_accumulo_skip_shell_tasks == True' on subsequent plays
dw_accumulo_skip_shell_tasks: False
dw_accumulo_shell_commands:
- "createnamespace datawave"
- "config -s table.classpath.context={{ dw_accumulo_vfs_context_name }}"
- "setauths -s {{ dw_accumulo_root_auths | join(',') }}"
- "createtable {{ dw_query_metrics_meta_table }}"
- "createtable {{ dw_query_metrics_shard_table }}"
# Ingest
dw_ingest_dir: "datawave-{{ dw_build_profile }}-{{ dw_project_version }}"
dw_ingest_home: "{{ install_dir }}/{{ dw_ingest_dir }}"
dw_ingest_tarball: "{{ dw_ingest_dir }}-dist.tar.gz"
dw_ingest_tarball_path: "{{ dw_clone_dir }}/warehouse/assemble/datawave/target/{{ dw_ingest_tarball }}"
dw_num_shards: "{{ groups['workers'] | length }}"
dw_all_ingest_data_types: [wikipedia,myjson]
dw_hdfs_base_dir: "/datawave/ingest"
dw_yarn_resource_manager: "{{ groups['resourcemanager'][0] }}:8032"
dw_warehouse_hadoop_conf_dir: "{{ hadoop_home }}/etc/hadoop"
dw_ingest_hadoop_conf_dir: "{{ dw_warehouse_hadoop_conf_dir }}"
dw_mapred_ingest_opts: "-useInlineCombiner -ingestMetricsDisabled"
dw_ingest_log_dir: "{{ dw_ingest_home }}/logs"
dw_ingest_flag_dir: "{{ dw_ingest_home }}/flags"
dw_flag_metrics_dir: "{{ dw_ingest_home }}/flagMetrics"
dw_ingest_lockfile_dir: "{{ dw_ingest_home }}/locks"
dw_flag_maker_configs: "{{ dw_ingest_home }}/config/flag-maker-live.xml,"
dw_event_discard_interval: 0
dw_ingest_secrets_file: "{{ dw_ingest_home }}/bin/secrets.sh"
dw_job_cache_replication: "{{ groups['workers'] | length }}"
dw_edge_definition_file: "config/edge-definitions.xml"
# Live Ingest Jobs (Map-only, batch mutations written directly to Accumulo)
dw_live_max_ingest_jobs: 5
dw_live_max_files_per_job: 5
dw_live_ingest_data_types: "{{ dw_all_ingest_data_types | join(',') }}"
dw_live_queue_name: liveIngestQueue
dw_live_child_map_max_memory_mb: 2048
# Bulk Ingest Jobs (Output rfiles from reduce phase, bulk import into Accumulo)
dw_bulk_max_ingest_jobs: 5
dw_bulk_max_files_per_job: 5
dw_bulk_ingest_data_types: ""
dw_bulk_queue_name: bulkIngestQueue
dw_bulk_child_map_max_memory_mb: 2048
dw_bulk_child_reduce_max_memory_mb: 2048
dw_bulk_ingest_reducers: "{{ dw_num_shards }}"
dw_bulk_num_map_loaders: 1
# Web Services
dw_web_dir: "datawave-ws-deploy-application-{{ dw_project_version }}"
dw_web_home: "{{ install_dir }}/{{ dw_web_dir }}"
dw_web_tarball: "{{ dw_web_dir }}-{{ dw_build_profile }}.tar.gz"
dw_web_tarball_path: "{{ dw_clone_dir }}/web-services/deploy/application/target/{{ dw_web_tarball }}"
dw_wildfly_version: "17.0.1.Final"
dw_wildfly_home: "{{ install_dir }}/wildfly-{{ dw_wildfly_version }}"
dw_wildfly_tarball: "wildfly-{{ dw_wildfly_version }}.tar.gz"
dw_wildfly_sha256: "2812b43b6c8e61f2d764a12d5a11de79481461b8cac51434ea730ee5985d38de"
dw_wildfly_args: ""
dw_wildfly_secure_port: 8443
dw_keystore: "{{ dw_wildfly_home }}/standalone/configuration/certificates/testServer.p12"
dw_keystore_type: PKCS12
dw_keystore_password: ChangeIt
dw_truststore: "{{ dw_wildfly_home }}/standalone/configuration/certificates/ca.jks"
dw_truststore_type: JKS
dw_truststore_password: ChangeIt
dw_mysql_user_password: secret
dw_jboss_jmx_password: secret
dw_hornetq_cluster_password: secret
dw_hornetq_system_password: secret
dw_security_use_test_authservice: True
dw_test_user_subject_dn: "cn=test a. user, ou=example developers, o=example corp, c=us"
dw_test_user_issuer_dn: "cn=example corp ca, o=example corp, c=us"
dw_test_user_accumulo_auths: "{{ dw_accumulo_root_auths }}"
dw_test_user_webservice_roles: [AuthorizedUser,Administrator,JBossAdministrator]
dw_query_metrics_meta_table: "datawave.queryMetrics_m"
dw_query_metrics_shard_table: "datawave.queryMetrics_s"