diff --git a/hue/README.md b/hue/README.md new file mode 100644 index 000000000..51c056d88 --- /dev/null +++ b/hue/README.md @@ -0,0 +1,19 @@ +# Cloudera Hue + +Deploys the Cloudera Hue server allowing data exploration on Hive and S3 buckets. + +### Folders + +There is one main folder in the Hue component `hue` which contains the kustomize manifests. + +### Installation + +To install Hue add the following to the `kfctl` yaml file. + +```yaml +- kustomizeConfig: + repoRef: + name: manifests + path: hue/hue + name: hue +``` diff --git a/hue/hue/base/hive-site-xml-secret.yaml b/hue/hue/base/hive-site-xml-secret.yaml new file mode 100644 index 000000000..8af1a12b2 --- /dev/null +++ b/hue/hue/base/hive-site-xml-secret.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: hue-hive-site-xml +type: Opaque +stringData: + hive-site.xml: | + + + + + hive.server2.transport.mode + binary + Server transport mode. binary or http. + + + + hive.server2.thrift.http.port + 10000 + Port number when in HTTP mode. + + + + fs.s3a.awsAccessKeyId + $(s3_access_key_id) + + + + fs.s3a.awsSecretAccessKey + $(s3_secret_access_key) + + + + fs.s3a.endpoint + $(s3_endpoint) + + diff --git a/hue/hue/base/hue-db-init-job.yaml b/hue/hue/base/hue-db-init-job.yaml new file mode 100644 index 000000000..b778c71be --- /dev/null +++ b/hue/hue/base/hue-db-init-job.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: hue-db-init +spec: + parallelism: 1 + completions: 1 + template: + metadata: + name: hue-db-init + spec: + containers: + - name: db-init + image: hue + command: ["build/env/bin/hue", "migrate"] + imagePullPolicy: Always + volumeMounts: + - name: "hue-ini" + mountPath: /hue/desktop/conf/ + volumes: + - name: "hue-ini" + secret: + secretName: hue-ini + items: + - key: hue.ini + path: hue.ini + restartPolicy: OnFailure diff --git a/hue/hue/base/hue-dc.yaml b/hue/hue/base/hue-dc.yaml new file mode 100644 index 000000000..7760a4f2c --- /dev/null +++ b/hue/hue/base/hue-dc.yaml @@ -0,0 +1,70 @@ +--- +apiVersion: apps.openshift.io/v1 +kind: DeploymentConfig +metadata: + name: hue +spec: + selector: + deploymentconfig: hue + template: + metadata: + labels: + deploymentconfig: hue + spec: + containers: + - name: hue + image: hue + command: ["build/env/bin/hue", "runcpserver"] + imagePullPolicy: Always + resources: + requests: + cpu: 300m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: hue-secret + key: s3_access_key_id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: hue-secret + key: s3_secret_access_key + volumeMounts: + - name: "hue-ini" + mountPath: /hue/desktop/conf/ + - name: "hive-site-xml" + mountPath: /etc/hive/conf/ + livenessProbe: + failureThreshold: 4 + httpGet: + path: / + port: 8000 + scheme: HTTP + periodSeconds: 30 + initialDelaySeconds: 300 + successThreshold: 1 + timeoutSeconds: 5 + restartPolicy: Always + volumes: + - name: "hue-ini" + secret: + secretName: hue-ini + items: + - key: hue.ini + path: hue.ini + - name: "hive-site-xml" + secret: + secretName: hue-hive-site-xml + items: + - key: hive-site.xml + path: hive-site.xml + replicas: 1 + triggers: + - type: ConfigChange + strategy: + type: Rolling diff --git a/hue/hue/base/hue-ini-secret.yaml b/hue/hue/base/hue-ini-secret.yaml new file mode 100644 index 000000000..de25f157a --- /dev/null +++ b/hue/hue/base/hue-ini-secret.yaml @@ -0,0 +1,179 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: hue-ini +type: Opaque +stringData: + hue.ini: | + [desktop] + # Hide unused apps + app_blacklist=impala,security,jobbrowser,jobsub,pig,hbase,sqoop,zookeeper,spark,oozie,search + secret_key=thisISaSECRET_1234 + http_host=0.0.0.0 + http_port=8000 + is_hue_4=true + time_zone=America/Los_Angeles + django_debug_mode=false + dev=false + database_logging=false + send_dbug_messages=false + http_500_debug_mode=false + + [[django_admins]] + + [[custom]] + + [[ldap]] + [[[users]]] + + [[[groups]]] + + [[[ldap_servers]]] + + [[vcs]] + + [[database]] + engine=django.db.backends.mysql + host=hue-mysql.$(namespace).svc + port=3306 + user=$(database_user) + password=$(database_password) + name=$(database_name) + + [[session]] + + [[smtp]] + host=localhost + port=25 + user= + password= + tls=no + + [[kerberos]] + + [[oauth]] + + [[oidc]] + + [[metrics]] + + [notebook] + [[interpreters]] + [[[hive]]] + name=Hive + interface=hiveserver2 + + [[[impala]]] + name=Impala + interface=hiveserver2 + + [[[sparksql]]] + name=SparkSql + interface=hiveserver2 + + [[[text]]] + name=Text + interface=text + + [[[markdown]]] + name=Markdown + interface=text + + [dashboard] + is_enabled=true + has_sql_enabled=true + has_query_builder_enabled=true + has_report_enabled=true + + [[engines]] + [[[sql]]] + analytics=true + nesting=false + + [[[hive]]] + analytics=true + nesting=true + + [beeswax] + hive_server_host=thriftserver.$(namespace).svc + hive_server_port=10000 + auth_username=hue + auth_password=password + hive_conf_dir=/etc/hive/conf + + [[ssl]] + + [metastore] + enable_new_create_table=true + force_hs2_metadata=false + + [impala] + [[ssl]] + + [spark] + + [oozie] + + [filebrowser] + + [pig] + + [sqoop] + + [proxy] + + [hbase] + + [search] + + [libsolr] + + [indexer] + + [jobsub] + + [jobbrowser] + + [security] + + [zookeeper] + [[clusters]] + [[[default]]] + + [useradmin] + [[password_policy]] + + [liboozie] + oozie_url= + + [aws] + [[aws_accounts]] + [[[default]]] + access_key_id=$(s3_access_key_id) + secret_access_key=$(s3_secret_access_key) + is_secure=$(s3_is_secure) + region=$(s3_region) + host=$(s3_endpoint) + calling_format=boto.s3.connection.OrdinaryCallingFormat + allow_environment_credentials=false + + [libsentry] + + [libzookeeper] + + [librdbms] + [[databases]] + + [libsaml] + + [libopenid] + + [liboauth] + + [kafka] + [[kafka]] + + [metadata] + [[optimizer]] + [[navigator]] diff --git a/hue/hue/base/hue-mysql-dc.yaml b/hue/hue/base/hue-mysql-dc.yaml new file mode 100644 index 000000000..7e6980d8a --- /dev/null +++ b/hue/hue/base/hue-mysql-dc.yaml @@ -0,0 +1,114 @@ +--- +apiVersion: apps.openshift.io/v1 +kind: DeploymentConfig +metadata: + name: hue-mysql + annotations: + template.alpha.openshift.io/wait-for-ready: "true" +spec: + strategy: + type: Recreate + triggers: + - type: ImageChange + imageChangeParams: + automatic: true + containerNames: + - mysql + from: + kind: ImageStreamTag + - type: ConfigChange + replicas: 1 + selector: + deploymentconfig: hue-mysql + template: + metadata: + labels: + deploymentconfig: hue-mysql + spec: + containers: + - name: mysqld-exporter + image: mysqld-exporter + ports: + - containerPort: 9104 + protocol: TCP + env: + - name: DATA_SOURCE_NAME + valueFrom: + secretKeyRef: + name: "hue-mysqld-exporter-secret" + key: DATA_SOURCE_NAME + livenessProbe: + httpGet: + path: /metrics + port: 9104 + readinessProbe: + httpGet: + path: /metrics + port: 9104 + - name: mysql + image: mysql + ports: + - containerPort: 3306 + protocol: TCP + readinessProbe: + exec: + command: + - /bin/sh + - "-i" + - "-c" + - >- + MYSQL_PWD=$MYSQL_PASSWORD mysql -h 127.0.0.1 -u $MYSQL_USER -D + $MYSQL_DATABASE -e 'SELECT 1' + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: 3306 + timeoutSeconds: 1 + env: + - name: MYSQL_USER + valueFrom: + secretKeyRef: + name: "hue-mysql-secret" + key: database-user + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: "hue-mysql-secret" + key: database-password + - name: MYSQL_DATABASE + valueFrom: + secretKeyRef: + name: "hue-mysql-secret" + key: database-name + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: "hue-mysql-secret" + key: database-root-password + resources: + limits: + memory: "512Mi" + volumeMounts: + - name: "mysql" + mountPath: "/var/lib/mysql/data" + terminationMessagePath: "/dev/termination-log" + terminationMessagePolicy: File + imagePullPolicy: IfNotPresent + capabilities: {} + securityContext: + capabilities: {} + privileged: false + volumes: + - name: mysql + persistentVolumeClaim: + claimName: hue-mysql + restartPolicy: Always + dnsPolicy: ClusterFirst diff --git a/hue/hue/base/hue-mysql-pvc.yaml b/hue/hue/base/hue-mysql-pvc.yaml new file mode 100644 index 000000000..472c984b6 --- /dev/null +++ b/hue/hue/base/hue-mysql-pvc.yaml @@ -0,0 +1,13 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: hue-mysql + annotations: + volume.beta.kubernetes.io/storage-class: "$(storage_class)" +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1Gi" diff --git a/hue/hue/base/hue-mysql-secret.yaml b/hue/hue/base/hue-mysql-secret.yaml new file mode 100644 index 000000000..34db530ff --- /dev/null +++ b/hue/hue/base/hue-mysql-secret.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: hue-mysql-secret + annotations: + template.openshift.io/expose-username: "{.data['database-user']}" + template.openshift.io/expose-password: "{.data['database-password']}" + template.openshift.io/expose-database_name: "{.data['database-name']}" + template.openshift.io/expose-root_password: | + {.data['database-root-password']} +stringData: + database-user: datacatalog + database-password: datacatalog + database-name: datacatalog + database-root-password: root diff --git a/hue/hue/base/hue-mysql-service.yaml b/hue/hue/base/hue-mysql-service.yaml new file mode 100644 index 000000000..a025c8493 --- /dev/null +++ b/hue/hue/base/hue-mysql-service.yaml @@ -0,0 +1,20 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: hue-mysql + annotations: + template.openshift.io/expose-uri: | + 'mysql://{.spec.clusterIP}:{.spec.ports[?(.name=="mysql")].port}' +spec: + ports: + - name: hue-mysql + protocol: TCP + port: 3306 + targetPort: 3306 + selector: + deploymentconfig: hue-mysql + type: ClusterIP + sessionAffinity: None +status: + loadBalancer: {} diff --git a/hue/hue/base/hue-mysqld-exporter-secret.yaml b/hue/hue/base/hue-mysqld-exporter-secret.yaml new file mode 100644 index 000000000..d2f1dd490 --- /dev/null +++ b/hue/hue/base/hue-mysqld-exporter-secret.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: hue-mysqld-exporter-secret +stringData: + DATA_SOURCE_NAME: "root:$(database_root_password)@(localhost:3306)/$(database_name)" diff --git a/hue/hue/base/hue-route.yaml b/hue/hue/base/hue-route.yaml new file mode 100644 index 000000000..519ebef42 --- /dev/null +++ b/hue/hue/base/hue-route.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: hue +spec: + port: + targetPort: 8000-tcp + tls: + termination: edge + to: + kind: Service + name: hue + weight: 100 + wildcardPolicy: None diff --git a/hue/hue/base/hue-secret.yaml b/hue/hue/base/hue-secret.yaml new file mode 100644 index 000000000..cd78959bb --- /dev/null +++ b/hue/hue/base/hue-secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: hue-secret +stringData: + s3_access_key_id: SECRET_KEY_ID + s3_secret_access_key: SECRET_ACCESS_KEY diff --git a/hue/hue/base/hue-service.yaml b/hue/hue/base/hue-service.yaml new file mode 100644 index 000000000..69a9b7b7d --- /dev/null +++ b/hue/hue/base/hue-service.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: hue +spec: + selector: + deploymentconfig: hue + ports: + - name: 8000-tcp + port: 8000 + protocol: TCP + targetPort: 8000 diff --git a/hue/hue/base/kustomization.yaml b/hue/hue/base/kustomization.yaml new file mode 100644 index 000000000..f1294449c --- /dev/null +++ b/hue/hue/base/kustomization.yaml @@ -0,0 +1,133 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - hive-site-xml-secret.yaml + - hue-db-init-job.yaml + - hue-dc.yaml + - hue-ini-secret.yaml + - hue-mysql-dc.yaml + - hue-mysql-pvc.yaml + - hue-mysql-secret.yaml + - hue-mysql-service.yaml + - hue-mysqld-exporter-secret.yaml + - hue-route.yaml + - hue-secret.yaml + - hue-service.yaml + +commonLabels: + opendatahub.io/component: "true" + component.opendatahub.io/name: hue + component.opendatahub.io/part-of: datacatalog + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: + - name: hue-config + env: params.env + +vars: + - name: namespace + objref: + kind: Service + apiVersion: v1 + name: hue + fieldref: + fieldpath: metadata.namespace + - name: storage_class + objref: + kind: ConfigMap + apiVersion: v1 + name: hue-config + fieldref: + fieldpath: data.storage_class + - name: s3_endpoint + objref: + kind: ConfigMap + apiVersion: v1 + name: hue-config + fieldref: + fieldpath: data.s3_endpoint + - name: s3_region + objref: + kind: ConfigMap + apiVersion: v1 + name: hue-config + fieldref: + fieldpath: data.s3_region + - name: s3_is_secure + objref: + kind: ConfigMap + apiVersion: v1 + name: hue-config + fieldref: + fieldpath: data.s3_is_secure + - name: s3_access_key_id + objref: + kind: Secret + apiVersion: v1 + name: hue-secret + fieldref: + fieldpath: stringData.s3_access_key_id + - name: s3_secret_access_key + objref: + kind: Secret + apiVersion: v1 + name: hue-secret + fieldref: + fieldpath: stringData.s3_secret_access_key + - name: database_user + objref: + kind: Secret + apiVersion: v1 + name: hue-mysql-secret + fieldref: + fieldpath: stringData.database-user + - name: database_password + objref: + kind: Secret + apiVersion: v1 + name: hue-mysql-secret + fieldref: + fieldpath: stringData.database-password + - name: database_root_password + objref: + kind: Secret + apiVersion: v1 + name: hue-mysql-secret + fieldref: + fieldpath: stringData.database-root-password + - name: database_name + objref: + kind: Secret + apiVersion: v1 + name: hue-mysql-secret + fieldref: + fieldpath: stringData.database-name + +configurations: + - params.yaml + +images: + - name: hue + newName: shgriffi/hue + newTag: test + - name: mysql + newName: openshift/mysql + newTag: "5.7" + - name: mysqld-exporter + newName: prom/mysqld-exporter + +patchesJson6902: + - patch: |- + - op: add + path: /spec/triggers/0/imageChangeParams/from/name + value: mysql:5.7 + - op: add + path: /spec/triggers/0/imageChangeParams/from/namespace + value: openshift + target: + kind: DeploymentConfig + name: hue-mysql + version: v1 diff --git a/hue/hue/base/params.env b/hue/hue/base/params.env new file mode 100644 index 000000000..f9bd6b4bf --- /dev/null +++ b/hue/hue/base/params.env @@ -0,0 +1,4 @@ +storage_class= +s3_endpoint= +s3_region= +s3_is_secure=false diff --git a/hue/hue/base/params.yaml b/hue/hue/base/params.yaml new file mode 100644 index 000000000..377f1cc14 --- /dev/null +++ b/hue/hue/base/params.yaml @@ -0,0 +1,10 @@ +--- +varReference: + - path: stringData/hue.ini + kind: Secret + - path: stringData/hive-site.xml + kind: Secret + - path: stringData/DATA_SOURCE_NAME + kind: Secret + - path: metadata/annotations/volume.beta.kubernetes.io/storage-class + kind: PersistentVolumeClaim diff --git a/tests/basictests/hue.sh b/tests/basictests/hue.sh new file mode 100755 index 000000000..1dd3f3acb --- /dev/null +++ b/tests/basictests/hue.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +source $TEST_DIR/common + +MY_DIR=$(readlink -f `dirname "${BASH_SOURCE[0]}"`) + +source ${MY_DIR}/../util + +os::test::junit::declare_suite_start "$MY_SCRIPT" + +function test_hue() { + header "Testing ODH Hue installation" + os::cmd::expect_success "oc project ${ODHPROJECT}" + os::cmd::try_until_text "oc get deploymentconfig hue" "hue" $odhdefaulttimeout $odhdefaultinterval + os::cmd::try_until_text "oc get pods -l deploymentconfig=hue --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}'" "hue" $odhdefaulttimeout $odhdefaultinterval + runningpods=($(oc get pods -l deploymentconfig=hue --field-selector="status.phase=Running" -o jsonpath="{$.items[*].metadata.name}")) + os::cmd::expect_success_and_text "echo ${#runningpods[@]}" "1" +} + +test_hue + +os::test::junit::declare_suite_end