From db7018e527798ea87be0b9a8b2e349a47aef468f Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Thu, 27 Jul 2023 16:37:04 +0200 Subject: [PATCH 01/31] initialized pycsw service for geonode --- deployment/geonode/templates/_helpers.tpl | 4 + .../templates/geonode/geonode-env.yaml | 7 + .../geonode/templates/nginx/nginx-conf.yaml | 14 ++ .../geonode/templates/pycsw/pycsw-cfg.yaml | 69 ++++++++ .../geonode/templates/pycsw/pycsw-deploy.yaml | 78 +++++++++ .../geonode/templates/pycsw/pycsw-env.yaml | 13 ++ .../templates/pycsw/pycsw-mappings-py.yaml | 73 +++++++++ .../geonode/templates/pycsw/pycsw-svc.yaml | 14 ++ deployment/geonode/values.yaml | 150 ++++++++++++++---- 9 files changed, 394 insertions(+), 28 deletions(-) create mode 100644 deployment/geonode/templates/pycsw/pycsw-cfg.yaml create mode 100644 deployment/geonode/templates/pycsw/pycsw-deploy.yaml create mode 100644 deployment/geonode/templates/pycsw/pycsw-env.yaml create mode 100644 deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml create mode 100644 deployment/geonode/templates/pycsw/pycsw-svc.yaml diff --git a/deployment/geonode/templates/_helpers.tpl b/deployment/geonode/templates/_helpers.tpl index 2441c35..fc12707 100644 --- a/deployment/geonode/templates/_helpers.tpl +++ b/deployment/geonode/templates/_helpers.tpl @@ -16,6 +16,10 @@ {{ .Release.Name }}-{{ .Values.nginx.pod_name }} {{- end -}} +{{- define "pycsw_pod_name" -}} +{{ .Release.Name }}-{{ .Values.pycsw.pod_name }} +{{- end -}} + # Volume names {{- define "persistant_volume_name" -}} diff --git a/deployment/geonode/templates/geonode/geonode-env.yaml b/deployment/geonode/templates/geonode/geonode-env.yaml index 5c9f6fc..756af75 100644 --- a/deployment/geonode/templates/geonode/geonode-env.yaml +++ b/deployment/geonode/templates/geonode/geonode-env.yaml @@ -127,6 +127,13 @@ data: LDAP_USER_ATTR_MAP_LAST_NAME: {{ .Values.geonode.ldap.attr_map_last_name | quote }} LDAP_USER_ATTR_MAP_EMAIL_ADDR: {{ .Values.geonode.ldap.attr_map_email_addr | quote }} + # Configure PYCSW + {{ if .Values.pycsw.enabled }} + CATALOGUE_ENGINE: geonode.catalogue.backends.pycsw_http + CATALOGUE_URL: {{ .Values.pycsw.pod_name}} + {{ else }} + CATALOGUE_ENGINE: geonode.catalogue.backends.pycsw_local + {{ end }} # OAuth2 # TODO (mwall) implement OAUTH2 OAUTH2_API_KEY: "" diff --git a/deployment/geonode/templates/nginx/nginx-conf.yaml b/deployment/geonode/templates/nginx/nginx-conf.yaml index ef679e9..0c9ccab 100644 --- a/deployment/geonode/templates/nginx/nginx-conf.yaml +++ b/deployment/geonode/templates/nginx/nginx-conf.yaml @@ -110,6 +110,20 @@ data: proxy_http_version 1.1; } + {{ if .Values.pycsw.enabled }} + # external PYCSW forward + location {{ .Values.pycsw.endpoint }} { + # Using a variable is a trick to let Nginx start even if upstream host is not up yet + set $upstream {{ include "pycsw_pod_name" . }}:{{ .Values.pycsw.port }}; + proxy_pass http://$upstream; + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_http_version 1.1; + } + {{ end }} + location / { client_max_body_size {{ .Values.nginx.maxClientBodySize }}; diff --git a/deployment/geonode/templates/pycsw/pycsw-cfg.yaml b/deployment/geonode/templates/pycsw/pycsw-cfg.yaml new file mode 100644 index 0000000..25c3463 --- /dev/null +++ b/deployment/geonode/templates/pycsw/pycsw-cfg.yaml @@ -0,0 +1,69 @@ +{{ if .Values.pycsw.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-pycsw-pycsw-cfg + namespace: {{ .Release.Namespace }} +data: + pycsw.cfg: |+ + [server] + {{ if .Values.pycsw.config.server.home }} home={{ .Values.pycsw.config.server.home }}{{ end }} + {{ if .Values.pycsw.config.server.url }} url={{ .Values.pycsw.endpoint }}{{ end }} + {{ if .Values.pycsw.config.server.mimetype }} mimetype={{ .Values.pycsw.config.server.mimetype }}{{ end }} + {{ if .Values.pycsw.config.server.encoding }} encoding={{ .Values.pycsw.config.server.encoding }}{{ end }} + {{ if .Values.pycsw.config.server.language }} language={{ .Values.pycsw.config.server.language }}{{ end }} + {{ if .Values.pycsw.config.server.maxrecords }} maxrecords={{ .Values.pycsw.config.server.maxrecords }}{{ end }} + {{ if .Values.pycsw.config.server.loglevel }} loglevel={{ .Values.pycsw.config.server.loglevel }}{{ end }} + {{ if .Values.pycsw.config.server.logfile }} logfile={{ .Values.pycsw.config.server.logfile }}{{ end }} + {{ if .Values.pycsw.config.server.ogc_schemas_base }} ogc_schemas_base={{ .Values.pycsw.config.server.ogc_schemas_base }}{{ end }} + {{ if .Values.pycsw.config.server.federatedcatalogues }} federatedcatalogues={{ .Values.pycsw.config.server.federatedcatalogues }}{{ end }} + {{ if .Values.pycsw.config.server.pretty_print }} pretty_print={{ .Values.pycsw.config.server.pretty_print }}{{ end }} + {{ if .Values.pycsw.config.server.gzip_compresslevel }} gzip_compresslevel={{ .Values.pycsw.config.server.gzip_compresslevel }}{{ end }} + {{ if .Values.pycsw.config.server.domainquerytype }} domainquerytype={{ .Values.pycsw.config.server.domainquerytype }}{{ end }} + {{ if .Values.pycsw.config.server.domaincounts }} domaincounts={{ .Values.pycsw.config.server.domaincounts }}{{ end }} + {{ if .Values.pycsw.config.server.spatial_ranking }} spatial_ranking={{ .Values.pycsw.config.server.spatial_ranking }}{{ end }} + {{ if .Values.pycsw.config.server.profiles }} profiles={{ .Values.pycsw.config.server.profiles }}{{ end }} + {{ if .Values.pycsw.config.server.workers }} workers={{ .Values.pycsw.config.server.workers }}{{ end }} + {{ if .Values.pycsw.config.server.timeout }} timeout={{ .Values.pycsw.config.server.timeout }}{{ end }} + [manager] + {{ if .Values.pycsw.config.manager.transactions }} transactions={{ .Values.pycsw.config.manager.transactions }}{{ end }} + {{ if .Values.pycsw.config.manager.allowed_ips }} allowed_ips={{ .Values.pycsw.config.manager.allowed_ips }}{{ end }} + {{ if .Values.pycsw.config.manager.csw_harvest_pagesize }} csw_harvest_pagesize={{ .Values.pycsw.config.manager.csw_harvest_pagesize }}{{ end }} + [metadata:main] + {{ if .Values.pycsw.config.metadata.identification_title }} identification_title={{ .Values.pycsw.config.metadata.identification_title }}{{ end }} + {{ if .Values.pycsw.config.metadata.identification_abstract }} identification_abstract={{ .Values.pycsw.config.metadata.identification_abstract }}{{ end }} + {{ if .Values.pycsw.config.metadata.identification_keywords }} identification_keywords={{ .Values.pycsw.config.metadata.identification_keywords }}{{ end }} + {{ if .Values.pycsw.config.metadata.identification_keywords_type }} identification_keywords_type={{ .Values.pycsw.config.metadata.identification_keywords_type }}{{ end }} + {{ if .Values.pycsw.config.metadata.identification_fees }} identification_fees={{ .Values.pycsw.config.metadata.identification_fees }}{{ end }} + {{ if .Values.pycsw.config.metadata.identification_accessconstraints }} identification_accessconstraints={{ .Values.pycsw.config.metadata.identification_accessconstraints }}{{ end }} + {{ if .Values.pycsw.config.metadata.provider_name }} provider_name={{ .Values.pycsw.config.metadata.provider_name }}{{ end }} + {{ if .Values.pycsw.config.metadata.provider_url }} provider_url={{ .Values.pycsw.config.metadata.provider_url }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_name }} contact_name={{ .Values.pycsw.config.metadata.contact_name }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_position }} contact_position={{ .Values.pycsw.config.metadata.contact_position }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_address }} contact_address={{ .Values.pycsw.config.metadata.contact_address }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_city }} contact_city={{ .Values.pycsw.config.metadata.contact_city }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_stateorprovince }} contact_stateorprovince={{ .Values.pycsw.config.metadata.contact_stateorprovince }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_postalcode }} contact_postalcode={{ .Values.pycsw.config.metadata.contact_postalcode }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_country }} contact_country={{ .Values.pycsw.config.metadata.contact_country }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_phone }} contact_phone={{ .Values.pycsw.config.metadata.contact_phone }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_fax }} contact_fax={{ .Values.pycsw.config.metadata.contact_fax }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_email }} contact_email={{ .Values.pycsw.config.metadata.contact_email }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_url }} contact_url={{ .Values.pycsw.config.metadata.contact_url }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_hours }} contact_hours={{ .Values.pycsw.config.metadata.contact_hours }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_instructions }} contact_instructions={{ .Values.pycsw.config.metadata.contact_instructions }}{{ end }} + {{ if .Values.pycsw.config.metadata.contact_role }} contact_role={{ .Values.pycsw.config.metadata.contact_role }}{{ end }} + [repository] + {{ if .Values.pycsw.config.repository.mappings }} mappings={{ .Values.pycsw.config.repository.mappings }}{{ end }} + {{ if .Values.pycsw.config.repository.table }} table={{ .Values.pycsw.config.repository.table }}{{ end }} + {{ if .Values.pycsw.config.repository.filter }} filter={{ .Values.pycsw.config.repository.filter }}{{ end }} + [metadata:inspire] + {{ if .Values.pycsw.config.inspire.enabled }} enabled={{ .Values.pycsw.config.inspire.enabled }}{{ end }} + {{ if .Values.pycsw.config.inspire.languages_supported }} languages_supported={{ .Values.pycsw.config.inspire.languages_supported }}{{ end }} + {{ if .Values.pycsw.config.inspire.default_language }} default_language={{ .Values.pycsw.config.inspire.default_language }}{{ end }} + {{ if .Values.pycsw.config.inspire.date }} date={{ .Values.pycsw.config.inspire.date }}{{ end }} + {{ if .Values.pycsw.config.inspire.gemet_keywords }} gemet_keywords={{ .Values.pycsw.config.inspire.gemet_keywords }}{{ end }} + {{ if .Values.pycsw.config.inspire.conformity_service }} conformity_service={{ .Values.pycsw.config.inspire.conformity_service }}{{ end }} + {{ if .Values.pycsw.config.inspire.contact_name }} contact_name={{ .Values.pycsw.config.inspire.contact_name }}{{ end }} + {{ if .Values.pycsw.config.inspire.contact_email }} contact_email={{ .Values.pycsw.config.inspire.contact_email }}{{ end }} + {{ if .Values.pycsw.config.inspire.temp_extent }} temp_extent={{ .Values.pycsw.config.inspire.temp_extent }}{{ end }} +{{ end }} \ No newline at end of file diff --git a/deployment/geonode/templates/pycsw/pycsw-deploy.yaml b/deployment/geonode/templates/pycsw/pycsw-deploy.yaml new file mode 100644 index 0000000..12bf41d --- /dev/null +++ b/deployment/geonode/templates/pycsw/pycsw-deploy.yaml @@ -0,0 +1,78 @@ +{{ if .Values.pycsw.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: "{{ include "pycsw_pod_name" . }}" + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.pycsw.replicaCount }} + selector: + matchLabels: + org.geonode.instance: "{{ include "pycsw_pod_name" . }}" + serviceName: "{{ include "pycsw_pod_name" . }}" + strategy: + type: Recreate + template: + metadata: + labels: + org.geonode.instance: "{{ include "pycsw_pod_name" . }}" + annotations: + checksum/pycsw-env: {{ include (print $.Template.BasePath "/pycsw/pycsw-env.yaml") . | sha256sum }} + checksum/pycsw-cfg: {{ include (print $.Template.BasePath "/pycsw/pycsw-cfg.yaml") . | sha256sum }} + checksum/pycsw-mappings-py: {{ include (print $.Template.BasePath "/pycsw/pycsw-mappings-py.yaml") . | sha256sum }} + spec: + terminationGracePeriodSeconds: 3 + containers: + - name: {{ .Values.pycsw.container_name }} + image: "{{ .Values.pycsw.image.name }}:{{ .Values.pycsw.image.tag }}" + + envFrom: + - configMapRef: + name: {{ include "pycsw_pod_name" . }}-env + + env: + - name: PYCSW_SERVER_URL + value: "{{ .Values.geonode.ingress.externalDomain}}{{ .Values.pycsw.endpoint }}" + - name: GEONODE_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.postgres.geonodedatabase }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do + key: password + - name: PYCSW_REPOSITORY_DATABASE_URI + value: "postgis://$(GEONODE_DATABASE):$(GEONODE_DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(GEONODE_DATABASE)" + + ports: + - containerPort: {{ .Values.pycsw.port }} + + volumeMounts: + - name: pycsw-pycsw-cfg + mountPath: "/etc/pycsw/pycsw.cfg" + subPath: pycsw.cfg + readOnly: true + - name: pycsw-mappings-py + mountPath: {{ .Values.pycsw.config.repository.mappings }} + subPath: mappings.py + readOnly: true + resources: + requests: + memory: {{ .Values.pycsw.resources.requests.memory }} + cpu: {{ .Values.pycsw.resources.requests.cpu }} + limits: + memory: {{ .Values.pycsw.resources.limits.memory }} + cpu: {{ .Values.pycsw.resources.limits.cpu }} + volumes: + - name: pycsw-pycsw-cfg + configMap: + name: {{ .Release.Name }}-pycsw-pycsw-cfg + defaultMode: 0744 + items: + - key: pycsw-pycsw-cfg + path: "pycsw-pycsw-cfg" + - name: pycsw-mappings-py + configMap: + name: {{ .Release.Name }}-pycsw-mappings-py + defaultMode: 0744 + items: + - key: pycsw-mappings-py + path: "pycsw-mappings-py" +{{ end }} diff --git a/deployment/geonode/templates/pycsw/pycsw-env.yaml b/deployment/geonode/templates/pycsw/pycsw-env.yaml new file mode 100644 index 0000000..c1c38a7 --- /dev/null +++ b/deployment/geonode/templates/pycsw/pycsw-env.yaml @@ -0,0 +1,13 @@ +{{ if .Values.pycsw.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "pycsw_pod_name" . }}-env + namespace: {{ .Release.Namespace }} +data: + # Database Settings + DATABASE_HOST: "{{ include "postgres_pod_name" . }}" + DATABASE_PORT: "{{ include "database_port" .}}" + GEONODE_DATABASE: {{ .Values.postgres.geonodedatabase | quote }} + GEONODE_DATABASE_SCHEMA: {{ .Values.postgres.schema }} +{{ end }} diff --git a/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml b/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml new file mode 100644 index 0000000..41c1be2 --- /dev/null +++ b/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml @@ -0,0 +1,73 @@ +{{ if .Values.pycsw.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-pycsw-mappings-py + namespace: {{ .Release.Namespace }} +data: + mappings.py: | + # copied from 4.1.x: https://github.com/GeoNode/geonode/blob/master/geonode/catalogue/backends/pycsw_local_mappings.py + MD_CORE_MODEL = { + "typename": "pycsw:CoreMetadata", + "outputschema": "http://pycsw.org/metadata", + "mappings": { + "pycsw:Identifier": "uuid", + "pycsw:Typename": "csw_typename", + "pycsw:Schema": "csw_schema", + "pycsw:MdSource": "csw_mdsource", + "pycsw:InsertDate": "csw_insert_date", + "pycsw:XML": "metadata_xml", + "pycsw:AnyText": "csw_anytext", + "pycsw:Language": "language", + "pycsw:Title": "title", + "pycsw:Abstract": "raw_abstract", + "pycsw:Keywords": "keyword_csv", + "pycsw:KeywordType": "keywordstype", + "pycsw:Format": "spatial_representation_type_string", + "pycsw:Source": "source", + "pycsw:Date": "date", + "pycsw:Modified": "date", + "pycsw:Type": "csw_type", + "pycsw:BoundingBox": "csw_wkt_geometry", + "pycsw:CRS": "csw_crs", + "pycsw:AlternateTitle": "alternate", + "pycsw:RevisionDate": "date", + "pycsw:CreationDate": "date", + "pycsw:PublicationDate": "date", + "pycsw:Organization": "organizationname", + "pycsw:OrganizationName": "organizationname", + "pycsw:SecurityConstraints": "securityconstraints", + "pycsw:ParentIdentifier": "parentidentifier", + "pycsw:TopicCategory": "topiccategory", + "pycsw:ResourceLanguage": "language", + "pycsw:GeographicDescriptionCode": "geodescode", + "pycsw:Denominator": "denominator", + "pycsw:DistanceValue": "distancevalue", + "pycsw:DistanceUOM": "distanceuom", + "pycsw:TempExtent_begin": "temporal_extent_start", + "pycsw:TempExtent_end": "temporal_extent_end", + "pycsw:ServiceType": "servicetype", + "pycsw:ServiceTypeVersion": "servicetypeversion", + "pycsw:Operation": "operation", + "pycsw:CouplingType": "couplingtype", + "pycsw:OperatesOn": "operateson", + "pycsw:OperatesOnIdentifier": "operatesonidentifier", + "pycsw:OperatesOnName": "operatesoname", + "pycsw:Degree": "degree", + "pycsw:AccessConstraints": "restriction_code", + "pycsw:OtherConstraints": "raw_constraints_other", + "pycsw:Classification": "classification", + "pycsw:ConditionApplyingToAccessAndUse": "conditionapplyingtoaccessanduse", + "pycsw:Lineage": "lineage", + "pycsw:ResponsiblePartyRole": "responsiblepartyrole", + "pycsw:SpecificationTitle": "specificationtitle", + "pycsw:SpecificationDate": "specificationdate", + "pycsw:SpecificationDateType": "specificationdatetype", + "pycsw:Creator": "creator", + "pycsw:Publisher": "publisher", + "pycsw:Contributor": "contributor", + "pycsw:Relation": "relation", + "pycsw:Links": "download_links", + }, + } +{{ end }} \ No newline at end of file diff --git a/deployment/geonode/templates/pycsw/pycsw-svc.yaml b/deployment/geonode/templates/pycsw/pycsw-svc.yaml new file mode 100644 index 0000000..009007d --- /dev/null +++ b/deployment/geonode/templates/pycsw/pycsw-svc.yaml @@ -0,0 +1,14 @@ +{{- if .Values.pycsw.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: "{{ include "pycsw_pod_name" . }}" + namespace: {{ .Release.Namespace }} +spec: + selector: + org.geonode.instance: "{{ include "pycsw_pod_name" . }}" + ports: + - targetPort: 8000 + port: {{ .Values.pycsw.port }} + name: pycsw +{{- end }} diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index 13f92a5..043923b 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -316,14 +316,6 @@ geoserver: # -- limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) cpu: 2 -# MEMCACHED CONFIGURATION -# https://artifacthub.io/packages/helm/bitnami/memcached -memcached: - # -- memcached replica. Loadbalanaced via kubernetes. (only one entry in django settings.py) im memcached is activated under geonode.memcached.enabled this takes place - architecture: high-availability - replicaCount: 1 - - # CONFIGURATION FOR NGINX DEPLOYMENT # MAY MOVE TO CHART nginx: @@ -353,27 +345,99 @@ nginx: # -- limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) cpu: "800m" -# -- VALUES DEFINITION https://github.com/bitnami/charts/blob/master/bitnami/rabbitmq/values.yaml -rabbitmq: - enabled: true +pycsw: + # -- enable single pycsw pod + enabled: True + # -- pycsw pod name + pod_name: pysw + # -- pycsw container replicas replicaCount: 1 - auth: - username: rabbituser - password: rabbitpassword - erlangCookie: jixYBsiZ9RivaLXC02pTwGjvIo0nHtVu - persistence: - enabled: False + # -- pycsw container name + container_name: pycsw + image: + # -- pycsw docker image + name: geopython/pycsw + # -- pycsw docker image tag + tag: '2.6.0' + # -- pycsw endpoint port + port: 8000 + # -- pycsw url below geonode.ingress.externalDomain + endpoint: /catalogue/csw_test + resources: + requests: + # -- requested memory as in resource.requests.memory (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) + memory: "1Gi" + # -- requested cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) + cpu: "500m" + limits: + # -- limits memory as in resource.limits.memory (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) + memory: "1Gi" + # -- limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) + cpu: "500m" + + config: + server: + home: /home/pycsw + mimetype: application/xml; charset=UTF-8 + encoding: UTF-8 + language: en-US + maxrecords: 10 + loglevel: DEBUG + # logfile: " " + # ogc_schemas_base: http://foo + # federatedcatalogues: http://catalog.data.gov/csw + # pretty_print: true + # gzip_compresslevel: 8 + # domainquerytype: range + # domaincounts: true + # spatial_ranking: true + profiles: apiso + # workers: 2 + timeout: 30 + manager: + transactions: "false" + # allowed_ips: 127.0.0.1 + # csw_harvest_pagesize: 10 + metadata: + identification_title: GeoNode pycsw Geospatial Catalogue + identification_abstract: GeoNode-k8s pycsw Geospatial Catalogue + identification_keywords: catalogue,discovery,metadata, geonode + identification_keywords_type: theme + identification_fees: None + identification_accessconstraints: None + provider_name: GeoNode Kubernetes + provider_url: https://pycsw.org/ + contact_name: Lastname, Firstname + contact_position: Position Title + contact_address: Mailing Address + contact_city: City + contact_stateorprovince: Administrative Area + contact_postalcode: Zip or Postal Code + contact_country: Country + contact_phone: +xx-xxx-xxx-xxxx + contact_fax: +xx-xxx-xxx-xxxx + contact_email: Email Address + contact_url: Contact URL + contact_hours: Hours of Service + contact_instructions: During hours of service. Off on weekends. + contact_role: pointOfContact + + repository: + table: base_resourcebase + mappings: /etc/pycsw/mappings.py + # filter: type = 'http://purl.org/dc/dcmitype/Dataset' + + inspire: + enabled: "true" + languages_supported: eng,gre + default_language: eng + date: YYYY-MM-DD + gemet_keywords: Utility and governmental services + conformity_service: notEvaluated + contact_name: Organization Name + contact_email: Email Address + temp_extent: YYYY-MM-DD/YYYY-MM-DD - requests: - # -- requested memory as in resource.requests.memory (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) - memory: "1Gi" - # -- requested cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) - cpu: "500m" - limits: - # -- limits memory as in resource.limits.memory (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) - memory: "1Gi" - # -- limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) - cpu: "750m" postgres: # -- pod name for postgres containers == teamID for mainifest @@ -390,7 +454,6 @@ postgres: # infos @ https://postgres-operator.readthedocs.io/en/refactoring-sidecars/user/ # get password after creation via: kubectl get secret {{ .Release.name }}.{{ .Release.name }}-{{ container_name }}.credentials -o 'jsonpath={.data.password}' | base64 -d - # -- configuration for postgres operator database manifest operator_manifest: # -- Database storage size @@ -404,6 +467,37 @@ postgres: # CHART CONFIGURATIONS # ######################## +# MEMCACHED CONFIGURATION +# https://artifacthub.io/packages/helm/bitnami/memcached +memcached: + # -- memcached replica. Loadbalanaced via kubernetes. (only one entry in django settings.py) im memcached is activated under geonode.memcached.enabled this takes place + architecture: high-availability + replicaCount: 1 + +# -- VALUES DEFINITION https://github.com/bitnami/charts/blob/master/bitnami/rabbitmq/values.yaml +rabbitmq: + enabled: true + # -- rabbitmq raplica count + replicaCount: 1 + auth: + username: rabbituser + password: rabbitpassword + erlangCookie: jixYBsiZ9RivaLXC02pTwGjvIo0nHtVu + persistence: + enabled: False + + requests: + # -- requested memory as in resource.requests.memory (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) + memory: "1Gi" + # -- requested cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) + cpu: "500m" + limits: + # -- limits memory as in resource.limits.memory (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) + memory: "1Gi" + # -- limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) + cpu: "750m" + + # VALUES DEFINITION: https://github.com/zalando/postgres-operator/blob/master/charts/postgres-operator/values.yaml postgres-operator: # -- enable postgres-operator (this or postgresql.enabled NOT both ) From 4aa52b87207f442dcef5cbbdc3f008326088a004 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 11:40:27 +0200 Subject: [PATCH 02/31] added pycsw forwarding if external pycsw is activated --- deployment/geonode/templates/nginx/nginx-conf.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/deployment/geonode/templates/nginx/nginx-conf.yaml b/deployment/geonode/templates/nginx/nginx-conf.yaml index 0c9ccab..855137d 100644 --- a/deployment/geonode/templates/nginx/nginx-conf.yaml +++ b/deployment/geonode/templates/nginx/nginx-conf.yaml @@ -113,14 +113,13 @@ data: {{ if .Values.pycsw.enabled }} # external PYCSW forward location {{ .Values.pycsw.endpoint }} { - # Using a variable is a trick to let Nginx start even if upstream host is not up yet - set $upstream {{ include "pycsw_pod_name" . }}:{{ .Values.pycsw.port }}; - proxy_pass http://$upstream; - proxy_redirect off; + client_max_body_size {{ .Values.nginx.maxClientBodySize }}; + + proxy_pass http://{{ include "pycsw_pod_name" . }}:{{ .Values.pycsw.port }}; proxy_set_header Host $host; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_http_version 1.1; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; } {{ end }} From 5b70812cbd596b594450699b11ccdd3b5671fa35 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 11:40:49 +0200 Subject: [PATCH 03/31] get repository database from env var --- deployment/geonode/templates/pycsw/pycsw-cfg.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment/geonode/templates/pycsw/pycsw-cfg.yaml b/deployment/geonode/templates/pycsw/pycsw-cfg.yaml index 25c3463..6d17a03 100644 --- a/deployment/geonode/templates/pycsw/pycsw-cfg.yaml +++ b/deployment/geonode/templates/pycsw/pycsw-cfg.yaml @@ -2,13 +2,13 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ .Release.Name }}-pycsw-pycsw-cfg + name: {{ .Release.Name }}-pycsw-cfg namespace: {{ .Release.Namespace }} data: pycsw.cfg: |+ [server] {{ if .Values.pycsw.config.server.home }} home={{ .Values.pycsw.config.server.home }}{{ end }} - {{ if .Values.pycsw.config.server.url }} url={{ .Values.pycsw.endpoint }}{{ end }} + url={{ .Values.geonode.ingress.externalScheme}}://{{ .Values.geonode.ingress.externalDomain}}{{ .Values.pycsw.endpoint }} {{ if .Values.pycsw.config.server.mimetype }} mimetype={{ .Values.pycsw.config.server.mimetype }}{{ end }} {{ if .Values.pycsw.config.server.encoding }} encoding={{ .Values.pycsw.config.server.encoding }}{{ end }} {{ if .Values.pycsw.config.server.language }} language={{ .Values.pycsw.config.server.language }}{{ end }} @@ -53,6 +53,7 @@ data: {{ if .Values.pycsw.config.metadata.contact_instructions }} contact_instructions={{ .Values.pycsw.config.metadata.contact_instructions }}{{ end }} {{ if .Values.pycsw.config.metadata.contact_role }} contact_role={{ .Values.pycsw.config.metadata.contact_role }}{{ end }} [repository] + database=${PYCSW_REPOSITORY_DATABASE_URI} {{ if .Values.pycsw.config.repository.mappings }} mappings={{ .Values.pycsw.config.repository.mappings }}{{ end }} {{ if .Values.pycsw.config.repository.table }} table={{ .Values.pycsw.config.repository.table }}{{ end }} {{ if .Values.pycsw.config.repository.filter }} filter={{ .Values.pycsw.config.repository.filter }}{{ end }} From f6d356a32ce106818ab0fbb40b8aeafa6c6a40d8 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 11:41:37 +0200 Subject: [PATCH 04/31] fixed volume imports --- .../geonode/templates/pycsw/pycsw-deploy.yaml | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/deployment/geonode/templates/pycsw/pycsw-deploy.yaml b/deployment/geonode/templates/pycsw/pycsw-deploy.yaml index 12bf41d..c906f80 100644 --- a/deployment/geonode/templates/pycsw/pycsw-deploy.yaml +++ b/deployment/geonode/templates/pycsw/pycsw-deploy.yaml @@ -25,11 +25,9 @@ spec: containers: - name: {{ .Values.pycsw.container_name }} image: "{{ .Values.pycsw.image.name }}:{{ .Values.pycsw.image.tag }}" - envFrom: - configMapRef: name: {{ include "pycsw_pod_name" . }}-env - env: - name: PYCSW_SERVER_URL value: "{{ .Values.geonode.ingress.externalDomain}}{{ .Values.pycsw.endpoint }}" @@ -39,20 +37,18 @@ spec: name: {{ .Values.postgres.geonodedatabase }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do key: password - name: PYCSW_REPOSITORY_DATABASE_URI - value: "postgis://$(GEONODE_DATABASE):$(GEONODE_DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(GEONODE_DATABASE)" - + value: "postgresql://$(GEONODE_DATABASE):$(GEONODE_DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(GEONODE_DATABASE)" ports: - containerPort: {{ .Values.pycsw.port }} - volumeMounts: - - name: pycsw-pycsw-cfg + - name: pycsw-cfg mountPath: "/etc/pycsw/pycsw.cfg" subPath: pycsw.cfg readOnly: true - name: pycsw-mappings-py mountPath: {{ .Values.pycsw.config.repository.mappings }} - subPath: mappings.py - readOnly: true + subPath: pycsw-mappings.py + readOnly: true resources: requests: memory: {{ .Values.pycsw.resources.requests.memory }} @@ -61,18 +57,18 @@ spec: memory: {{ .Values.pycsw.resources.limits.memory }} cpu: {{ .Values.pycsw.resources.limits.cpu }} volumes: - - name: pycsw-pycsw-cfg + - name: pycsw-cfg configMap: - name: {{ .Release.Name }}-pycsw-pycsw-cfg + name: {{ .Release.Name }}-pycsw-cfg defaultMode: 0744 items: - - key: pycsw-pycsw-cfg - path: "pycsw-pycsw-cfg" + - key: pycsw.cfg + path: "pycsw.cfg" - name: pycsw-mappings-py configMap: name: {{ .Release.Name }}-pycsw-mappings-py defaultMode: 0744 items: - - key: pycsw-mappings-py - path: "pycsw-mappings-py" + - key: pycsw-mappings.py + path: "pycsw-mappings.py" {{ end }} From ce4512810dcd3f79a6ff836407ed33f77c0a2d76 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 11:41:53 +0200 Subject: [PATCH 05/31] renamed mapping filename --- deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml b/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml index 41c1be2..49c7502 100644 --- a/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml +++ b/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml @@ -5,7 +5,7 @@ metadata: name: {{ .Release.Name }}-pycsw-mappings-py namespace: {{ .Release.Namespace }} data: - mappings.py: | + pycsw-mappings.py: | # copied from 4.1.x: https://github.com/GeoNode/geonode/blob/master/geonode/catalogue/backends/pycsw_local_mappings.py MD_CORE_MODEL = { "typename": "pycsw:CoreMetadata", From dce608774fd13ddc93bf2d42249a227ff70c5b20 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 11:51:06 +0200 Subject: [PATCH 06/31] clean up --- deployment/geonode/values.yaml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index 043923b..52bfd9e 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -345,6 +345,7 @@ nginx: # -- limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) cpu: "800m" +# -- pycsw integration is based on https://github.com/geopython/pycsw/blob/master/docker/kubernetes pycsw: # -- enable single pycsw pod enabled: True @@ -362,7 +363,7 @@ pycsw: # -- pycsw endpoint port port: 8000 # -- pycsw url below geonode.ingress.externalDomain - endpoint: /catalogue/csw_test + endpoint: /catalogue/csw resources: requests: # -- requested memory as in resource.requests.memory (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) @@ -374,7 +375,7 @@ pycsw: memory: "1Gi" # -- limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) cpu: "500m" - + # -- pycsw config file parameters, see docs: https://docs.pycsw.org/_/downloads/en/latest/pdf/ config: server: home: /home/pycsw @@ -382,22 +383,11 @@ pycsw: encoding: UTF-8 language: en-US maxrecords: 10 - loglevel: DEBUG - # logfile: " " - # ogc_schemas_base: http://foo - # federatedcatalogues: http://catalog.data.gov/csw - # pretty_print: true - # gzip_compresslevel: 8 - # domainquerytype: range - # domaincounts: true - # spatial_ranking: true profiles: apiso # workers: 2 timeout: 30 manager: transactions: "false" - # allowed_ips: 127.0.0.1 - # csw_harvest_pagesize: 10 metadata: identification_title: GeoNode pycsw Geospatial Catalogue identification_abstract: GeoNode-k8s pycsw Geospatial Catalogue @@ -424,7 +414,7 @@ pycsw: repository: table: base_resourcebase - mappings: /etc/pycsw/mappings.py + mappings: /etc/pycsw/pycsw-mappings.py # filter: type = 'http://purl.org/dc/dcmitype/Dataset' inspire: From 777ef1c2d42d19bd6aab3e2022f287cef21a900b Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 11:57:01 +0200 Subject: [PATCH 07/31] added pycsw enabled for transparency reasons --- minikube-values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/minikube-values.yaml b/minikube-values.yaml index 7a79b36..03bac8a 100644 --- a/minikube-values.yaml +++ b/minikube-values.yaml @@ -31,6 +31,9 @@ geonode: email_verification: "False" authentication_method: username_email +pycsw: + enabled: True + postgres-operator-ui: enabled: False From e2df62dd3fda1ab1fcfd966f6b14e192a3bb16a3 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 14:54:21 +0200 Subject: [PATCH 08/31] pycsw docs --- docs/pycsw.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/pycsw.md diff --git a/docs/pycsw.md b/docs/pycsw.md new file mode 100644 index 0000000..dbc028e --- /dev/null +++ b/docs/pycsw.md @@ -0,0 +1,8 @@ +External pyCSW +-------------- + +GeoNode-k8s external pycsw container. This allows changes to pyCSW with having to change GeoNode codebase and allows scalability of the pyCSW service seperatly. The external pycsw service is by default enabled. To disable set: `.Values.pycsw.enabled = False` + +The pycsw configuration (pycsw.cfg) and mappings are defined in the values file: `[ .Values.pycsw.config , .Values.pycsw.mappings ]` + +The csw endpoint is default set to the one used within Geonode: `/catalogue/csw` From d62d6121b9af7385da02547d74271dd4e8918a35 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 15:05:26 +0200 Subject: [PATCH 09/31] fixed CATALOGUE_URL --- deployment/geonode/templates/geonode/geonode-env.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/geonode/templates/geonode/geonode-env.yaml b/deployment/geonode/templates/geonode/geonode-env.yaml index 756af75..94d7d65 100644 --- a/deployment/geonode/templates/geonode/geonode-env.yaml +++ b/deployment/geonode/templates/geonode/geonode-env.yaml @@ -130,10 +130,10 @@ data: # Configure PYCSW {{ if .Values.pycsw.enabled }} CATALOGUE_ENGINE: geonode.catalogue.backends.pycsw_http - CATALOGUE_URL: {{ .Values.pycsw.pod_name}} + CATALOGUE_URL: "{{ include "pycsw_pod_name" . }}:{{ .Values.pycsw.port }}" {{ else }} CATALOGUE_ENGINE: geonode.catalogue.backends.pycsw_local - {{ end }} + {{ end}} # OAuth2 # TODO (mwall) implement OAUTH2 OAUTH2_API_KEY: "" From 33181b5f35a4eec9ce8e8ccec32477d806708950 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 15:05:42 +0200 Subject: [PATCH 10/31] added compression --- deployment/geonode/templates/nginx/nginx-conf.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deployment/geonode/templates/nginx/nginx-conf.yaml b/deployment/geonode/templates/nginx/nginx-conf.yaml index 855137d..cf65dec 100644 --- a/deployment/geonode/templates/nginx/nginx-conf.yaml +++ b/deployment/geonode/templates/nginx/nginx-conf.yaml @@ -115,6 +115,15 @@ data: location {{ .Values.pycsw.endpoint }} { client_max_body_size {{ .Values.nginx.maxClientBodySize }}; + # compression + gzip on; + gzip_types + text/xml + text/plain + application/xml + application/xml+rss + application/json; + proxy_pass http://{{ include "pycsw_pod_name" . }}:{{ .Values.pycsw.port }}; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; From 305173937439094d4c9d3c8f3caa658acf661a60 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 15:07:24 +0200 Subject: [PATCH 11/31] moved pycsw mappings to values --- .../templates/pycsw/pycsw-mappings-py.yaml | 66 +------------------ deployment/geonode/values.yaml | 65 ++++++++++++++++++ 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml b/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml index 49c7502..e9fb666 100644 --- a/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml +++ b/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml @@ -5,69 +5,5 @@ metadata: name: {{ .Release.Name }}-pycsw-mappings-py namespace: {{ .Release.Namespace }} data: - pycsw-mappings.py: | - # copied from 4.1.x: https://github.com/GeoNode/geonode/blob/master/geonode/catalogue/backends/pycsw_local_mappings.py - MD_CORE_MODEL = { - "typename": "pycsw:CoreMetadata", - "outputschema": "http://pycsw.org/metadata", - "mappings": { - "pycsw:Identifier": "uuid", - "pycsw:Typename": "csw_typename", - "pycsw:Schema": "csw_schema", - "pycsw:MdSource": "csw_mdsource", - "pycsw:InsertDate": "csw_insert_date", - "pycsw:XML": "metadata_xml", - "pycsw:AnyText": "csw_anytext", - "pycsw:Language": "language", - "pycsw:Title": "title", - "pycsw:Abstract": "raw_abstract", - "pycsw:Keywords": "keyword_csv", - "pycsw:KeywordType": "keywordstype", - "pycsw:Format": "spatial_representation_type_string", - "pycsw:Source": "source", - "pycsw:Date": "date", - "pycsw:Modified": "date", - "pycsw:Type": "csw_type", - "pycsw:BoundingBox": "csw_wkt_geometry", - "pycsw:CRS": "csw_crs", - "pycsw:AlternateTitle": "alternate", - "pycsw:RevisionDate": "date", - "pycsw:CreationDate": "date", - "pycsw:PublicationDate": "date", - "pycsw:Organization": "organizationname", - "pycsw:OrganizationName": "organizationname", - "pycsw:SecurityConstraints": "securityconstraints", - "pycsw:ParentIdentifier": "parentidentifier", - "pycsw:TopicCategory": "topiccategory", - "pycsw:ResourceLanguage": "language", - "pycsw:GeographicDescriptionCode": "geodescode", - "pycsw:Denominator": "denominator", - "pycsw:DistanceValue": "distancevalue", - "pycsw:DistanceUOM": "distanceuom", - "pycsw:TempExtent_begin": "temporal_extent_start", - "pycsw:TempExtent_end": "temporal_extent_end", - "pycsw:ServiceType": "servicetype", - "pycsw:ServiceTypeVersion": "servicetypeversion", - "pycsw:Operation": "operation", - "pycsw:CouplingType": "couplingtype", - "pycsw:OperatesOn": "operateson", - "pycsw:OperatesOnIdentifier": "operatesonidentifier", - "pycsw:OperatesOnName": "operatesoname", - "pycsw:Degree": "degree", - "pycsw:AccessConstraints": "restriction_code", - "pycsw:OtherConstraints": "raw_constraints_other", - "pycsw:Classification": "classification", - "pycsw:ConditionApplyingToAccessAndUse": "conditionapplyingtoaccessanduse", - "pycsw:Lineage": "lineage", - "pycsw:ResponsiblePartyRole": "responsiblepartyrole", - "pycsw:SpecificationTitle": "specificationtitle", - "pycsw:SpecificationDate": "specificationdate", - "pycsw:SpecificationDateType": "specificationdatetype", - "pycsw:Creator": "creator", - "pycsw:Publisher": "publisher", - "pycsw:Contributor": "contributor", - "pycsw:Relation": "relation", - "pycsw:Links": "download_links", - }, - } + pycsw-mappings.py: {{- .Values.pycsw.mappings | toYaml | indent 1 }} {{ end }} \ No newline at end of file diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index 52bfd9e..386b9fa 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -375,7 +375,72 @@ pycsw: memory: "1Gi" # -- limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) cpu: "500m" + # copied from 4.1.x: https://github.com/GeoNode/geonode/blob/master/geonode/catalogue/backends/pycsw_local_mappings.py # -- pycsw config file parameters, see docs: https://docs.pycsw.org/_/downloads/en/latest/pdf/ + mappings: |- + MD_CORE_MODEL = { + "typename": "pycsw:CoreMetadata", + "outputschema": "http://pycsw.org/metadata", + "mappings": { + "pycsw:Identifier": "uuid", + "pycsw:Typename": "csw_typename", + "pycsw:Schema": "csw_schema", + "pycsw:MdSource": "csw_mdsource", + "pycsw:InsertDate": "csw_insert_date", + "pycsw:XML": "metadata_xml", + "pycsw:AnyText": "csw_anytext", + "pycsw:Language": "language", + "pycsw:Title": "title", + "pycsw:Abstract": "raw_abstract", + "pycsw:Keywords": "keyword_csv", + "pycsw:KeywordType": "keywordstype", + "pycsw:Format": "spatial_representation_type_string", + "pycsw:Source": "source", + "pycsw:Date": "date", + "pycsw:Modified": "date", + "pycsw:Type": "csw_type", + "pycsw:BoundingBox": "csw_wkt_geometry", + "pycsw:CRS": "csw_crs", + "pycsw:AlternateTitle": "alternate", + "pycsw:RevisionDate": "date", + "pycsw:CreationDate": "date", + "pycsw:PublicationDate": "date", + "pycsw:Organization": "organizationname", + "pycsw:OrganizationName": "organizationname", + "pycsw:SecurityConstraints": "securityconstraints", + "pycsw:ParentIdentifier": "parentidentifier", + "pycsw:TopicCategory": "topiccategory", + "pycsw:ResourceLanguage": "language", + "pycsw:GeographicDescriptionCode": "geodescode", + "pycsw:Denominator": "denominator", + "pycsw:DistanceValue": "distancevalue", + "pycsw:DistanceUOM": "distanceuom", + "pycsw:TempExtent_begin": "temporal_extent_start", + "pycsw:TempExtent_end": "temporal_extent_end", + "pycsw:ServiceType": "servicetype", + "pycsw:ServiceTypeVersion": "servicetypeversion", + "pycsw:Operation": "operation", + "pycsw:CouplingType": "couplingtype", + "pycsw:OperatesOn": "operateson", + "pycsw:OperatesOnIdentifier": "operatesonidentifier", + "pycsw:OperatesOnName": "operatesoname", + "pycsw:Degree": "degree", + "pycsw:AccessConstraints": "restriction_code", + "pycsw:OtherConstraints": "raw_constraints_other", + "pycsw:Classification": "classification", + "pycsw:ConditionApplyingToAccessAndUse": "conditionapplyingtoaccessanduse", + "pycsw:Lineage": "lineage", + "pycsw:ResponsiblePartyRole": "responsiblepartyrole", + "pycsw:SpecificationTitle": "specificationtitle", + "pycsw:SpecificationDate": "specificationdate", + "pycsw:SpecificationDateType": "specificationdatetype", + "pycsw:Creator": "creator", + "pycsw:Publisher": "publisher", + "pycsw:Contributor": "contributor", + "pycsw:Relation": "relation", + "pycsw:Links": "download_links", + }, + } config: server: home: /home/pycsw From 2309fe8b49b1cd4751ff432db5a233be914b4051 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 31 Jul 2023 17:20:28 +0200 Subject: [PATCH 12/31] added liveness probe and init container to wait for geonode, to no create database table before geonode is ready --- .../geonode/templates/pycsw/pycsw-deploy.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/deployment/geonode/templates/pycsw/pycsw-deploy.yaml b/deployment/geonode/templates/pycsw/pycsw-deploy.yaml index c906f80..df7a9f6 100644 --- a/deployment/geonode/templates/pycsw/pycsw-deploy.yaml +++ b/deployment/geonode/templates/pycsw/pycsw-deploy.yaml @@ -22,6 +22,18 @@ spec: checksum/pycsw-mappings-py: {{ include (print $.Template.BasePath "/pycsw/pycsw-mappings-py.yaml") . | sha256sum }} spec: terminationGracePeriodSeconds: 3 + initContainers: + # Wait for GeoNode to be up and running, else there can be a race conddition where pycsw creates the database table, + # and lets crash init process of geonode + - name: pycsw-wait-for-geonode + image: alpine/curl + imagePullPolicy: IfNotPresent + command: ["/bin/sh","-c"] + args: ['while [ $(curl -ksw "%{http_code}" "$GEONODE_ENDPOINT:8001" -o /dev/null) -ne 200 ]; do sleep 5; echo "health check failed . Waiting for GeoNode ($GEONODE_ENDPOINT:8001) ..."; done'] + env: + - name: GEONODE_ENDPOINT + value: "{{ include "geonode_pod_name" . }}" + containers: - name: {{ .Values.pycsw.container_name }} image: "{{ .Values.pycsw.image.name }}:{{ .Values.pycsw.image.tag }}" @@ -56,6 +68,13 @@ spec: limits: memory: {{ .Values.pycsw.resources.limits.memory }} cpu: {{ .Values.pycsw.resources.limits.cpu }} + livenessProbe: + httpGet: + path: / + port: {{ .Values.pycsw.port }} + initialDelaySeconds: 3 + periodSeconds: 10 + volumes: - name: pycsw-cfg configMap: From 8b88a011b0c260855eb3c445fa65953a39196096 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Tue, 1 Aug 2023 15:59:35 +0200 Subject: [PATCH 13/31] added http so the pycsw can be found by geonode --- deployment/geonode/templates/geonode/geonode-env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/geonode/templates/geonode/geonode-env.yaml b/deployment/geonode/templates/geonode/geonode-env.yaml index 94d7d65..ea66029 100644 --- a/deployment/geonode/templates/geonode/geonode-env.yaml +++ b/deployment/geonode/templates/geonode/geonode-env.yaml @@ -130,7 +130,7 @@ data: # Configure PYCSW {{ if .Values.pycsw.enabled }} CATALOGUE_ENGINE: geonode.catalogue.backends.pycsw_http - CATALOGUE_URL: "{{ include "pycsw_pod_name" . }}:{{ .Values.pycsw.port }}" + CATALOGUE_URL: "http://{{ include "pycsw_pod_name" . }}:{{ .Values.pycsw.port }}" {{ else }} CATALOGUE_ENGINE: geonode.catalogue.backends.pycsw_local {{ end}} From d4bb3da57a685466e2d7e5bcadcc0bbe94c41f7b Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Wed, 2 Aug 2023 15:51:26 +0200 Subject: [PATCH 14/31] prefix variable name of operator manifest --- .../templates/postgres/geonode-manifest.yaml | 4 +- deployment/geonode/values.yaml | 56 ++++++++++--------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/deployment/geonode/templates/postgres/geonode-manifest.yaml b/deployment/geonode/templates/postgres/geonode-manifest.yaml index 3708ab5..c1c9f84 100644 --- a/deployment/geonode/templates/postgres/geonode-manifest.yaml +++ b/deployment/geonode/templates/postgres/geonode-manifest.yaml @@ -5,8 +5,8 @@ metadata: spec: teamId: {{ .Release.Name | quote }} volume: - size: {{ .Values.postgres.operator_manifest.storageSize }} - numberOfInstances: {{ int .Values.postgres.operator_manifest.numberOfInstances }} + size: {{ .Values.postgres.postgres_operator_manifest.storageSize }} + numberOfInstances: {{ int .Values.postgres.postgres_operator_manifest.numberOfInstances }} users: {{ .Values.postgres.username }}: - superuser diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index 386b9fa..b55dc3e 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -493,31 +493,6 @@ pycsw: contact_email: Email Address temp_extent: YYYY-MM-DD/YYYY-MM-DD - -postgres: - # -- pod name for postgres containers == teamID for mainifest - pod_name: postgresql - # -- postgres username - username: postgres - # -- database schema - schema: public - # -- geonode database name - geonodedatabase: geonode - # -- geoserver database name - geodatabasename: geogeonode - # database passwords are set randomly - # infos @ https://postgres-operator.readthedocs.io/en/refactoring-sidecars/user/ - # get password after creation via: kubectl get secret {{ .Release.name }}.{{ .Release.name }}-{{ container_name }}.credentials -o 'jsonpath={.data.password}' | base64 -d - - # -- configuration for postgres operator database manifest - operator_manifest: - # -- Database storage size - storageSize: 3Gi - # -- number of database instances - numberOfInstances: 1 - # -- postgres version - postgres_version: 15 - ######################## # CHART CONFIGURATIONS # ######################## @@ -552,6 +527,33 @@ rabbitmq: # -- limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) cpu: "750m" +postgres: + # -- pod name for postgres containers == teamID for mainifest + pod_name: postgresql + # -- postgres username + username: postgres + # -- database schema + schema: public + # -- geonode database name + geonodedatabase: geonode + # -- geoserver database name + geodatabasename: geogeonode + # database passwords are set randomly + # infos @ https://postgres-operator.readthedocs.io/en/refactoring-sidecars/user/ + # get password after creation via: kubectl get secret {{ .Release.name }}.{{ .Release.name }}-{{ container_name }}.credentials -o 'jsonpath={.data.password}' | base64 -d + + # -- configuration for postgres operator database manifest + postgres_operator_manifest: + # -- Database storage size + storageSize: 3Gi + # -- number of database instances + numberOfInstances: 1 + # -- postgres version + postgres_version: 15 + +######################## +# CHART CONFIGURATIONS # +######################## # VALUES DEFINITION: https://github.com/zalando/postgres-operator/blob/master/charts/postgres-operator/values.yaml postgres-operator: @@ -560,8 +562,8 @@ postgres-operator: # -- ??? operatorApiUrl: "http://{{ .Release.Name }}-postgres-operator:8080" configLoggingRestApi: - # -- REST API listener listens to this port - api_port: 8080 + # -- REST API listener listens to this port + api_port: 8080 # -- postgress pv storageclass storageClass: # -- not setting the podServiceAccount name will leed to generation of this name. This allows to run multiple postgres-operators in a single kubernetes cluster. just seperating them by namespace. From c93be0eeb92da37161835250743be1a549753cef Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Mon, 31 Jul 2023 14:44:33 +0200 Subject: [PATCH 15/31] Use value configured in values.yaml --- deployment/geonode/templates/geonode/geonode-pvc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/geonode/templates/geonode/geonode-pvc.yaml b/deployment/geonode/templates/geonode/geonode-pvc.yaml index 1100c3f..63f7808 100644 --- a/deployment/geonode/templates/geonode/geonode-pvc.yaml +++ b/deployment/geonode/templates/geonode/geonode-pvc.yaml @@ -5,7 +5,7 @@ metadata: namespace: {{ .Release.Namespace }} spec: accessModes: - - ReadWriteMany + - {{ .Values.global.accessMode }} storageClassName: {{ .Values.global.storageClass }} resources: requests: From 15880c6b79bb7ba0fbdc9c3ec38df6bfc7c80aac Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Mon, 31 Jul 2023 14:56:44 +0200 Subject: [PATCH 16/31] Configure cert-manager acme section --- .../templates/nginx/nginx-ingress.yaml | 34 +++++++++++++++++-- deployment/geonode/values.yaml | 11 +++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/deployment/geonode/templates/nginx/nginx-ingress.yaml b/deployment/geonode/templates/nginx/nginx-ingress.yaml index a3ca1d0..be87592 100644 --- a/deployment/geonode/templates/nginx/nginx-ingress.yaml +++ b/deployment/geonode/templates/nginx/nginx-ingress.yaml @@ -2,14 +2,18 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: "{{ include "nginx_pod_name" . }}-ingress" - {{ if ( .Values.geonode.ingress.addNginxIngressAnnotation) }} annotations: + description: Configures routes for external access + {{ if (eq .Values.geonode.acme.enabled true) }} + cert-manager.io/issuer: letsencrypt + acme.cert-manager.io/http01-edit-in-place: "true" + {{ end}} + {{ if ( .Values.geonode.ingress.addNginxIngressAnnotation) }} nginx.ingress.kubernetes.io/proxy-body-size: "2g" nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" nginx.ingress.kubernetes.io/proxy-read-timeout: "600" nginx.ingress.kubernetes.io/proxy-send-timeout: "600" - - {{ end }} + {{ end }} spec: ingressClassName: {{ .Values.geonode.ingress.ingressClassName }} {{ if (eq .Values.geonode.ingress.externalScheme "https") }} @@ -29,3 +33,27 @@ spec: name: "{{ include "nginx_pod_name" . }}" port: number: {{ .Values.geonode.ingress.externalPort }} + +--- + + +{{if (eq .Values.geonode.acme.enabled true) }} + +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: letsencrypt +spec: + acme: + email: {{ .Values.geonode.acme.email }} + server: {{ .Values.geonode.acme.stageUrl }} + privateKeySecretRef: + name: letsencrypt + solvers: + - selector: + dnsNames: + - {{ .Values.geonode.ingress.externalDomain }} + http01: + ingress: + ingressClassName: {{ .Values.geonode.ingress.ingressClassName }} +{{ end }} \ No newline at end of file diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index b55dc3e..88f72ba 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -56,9 +56,18 @@ geonode: externalDomain: geonode # -- external ingress port externalPort: 80 - # -- tls certificate for geonode ingress https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/. is used when geonode.ingress.externalScheme is set to https + # -- tls certificate for geonode ingress https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ (for the use of cert-manager, configure the acme section properly). is used when geonode.ingress.externalScheme is set to https tlsSecret: geonode-tls-secret + acme: + # -- enables cert-manager to do ACME challenges (aka certificates via letsencrypt) + enabled: False + # -- the email to be used to gain certificates + email: support@example.com + # -- ACME staging environment (use acme-staging to avoid running into rate limits) + #stageUrl: https://acme-v02.api.letsencrypt.org/directory + stageUrl: https://acme-staging-v02.api.letsencrypt.org/directory + sentry: # -- enable sentry integration for geonode enabled: False From d025237aae9aa4e8c70c95c928e4b90cdf810bd2 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Mon, 31 Jul 2023 16:45:04 +0200 Subject: [PATCH 17/31] Update dependencies --- deployment/geonode/Chart.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/deployment/geonode/Chart.yaml b/deployment/geonode/Chart.yaml index c7edc38..00c375a 100644 --- a/deployment/geonode/Chart.yaml +++ b/deployment/geonode/Chart.yaml @@ -6,19 +6,19 @@ home: https://github.com/zalf-rdm/geonode-k8s sources: - https://github.com/zalf-rdm/geonode-k8s dependencies: -- name: postgres-operator-ui - version: 1.9.0 - repository: https://opensource.zalando.com/postgres-operator/charts/postgres-operator-ui/ - condition: postgres-operator-ui.enabled -- name: postgres-operator - version: 1.9.0 - repository: https://opensource.zalando.com/postgres-operator/charts/postgres-operator/ - condition: postgres-operator.enabled -- name: rabbitmq - version: ~10.1.7 - repository: https://charts.bitnami.com/bitnami - condition: rabbitmq.enabled -- name: memcached - repository: https://charts.bitnami.com/bitnami - condition: geonode.memcached.enaled - version: ~6.x.x + - name: postgres-operator-ui + version: ~1.10.0 + repository: https://opensource.zalando.com/postgres-operator/charts/postgres-operator-ui/ + condition: postgres-operator-ui.enabled + - name: postgres-operator + version: ~1.10.0 + repository: https://opensource.zalando.com/postgres-operator/charts/postgres-operator/ + condition: postgres-operator.enabled + - name: rabbitmq + version: ~10.1.7 + repository: https://charts.bitnami.com/bitnami + condition: rabbitmq.enabled + - name: memcached + repository: https://charts.bitnami.com/bitnami + condition: geonode.memcached.enaled + version: ~6.x.x From 40a623f8c034303fb216aa30f292fb0a26e4026e Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 1 Aug 2023 11:52:52 +0200 Subject: [PATCH 18/31] Use postgres-operator 1.9.0 --- deployment/geonode/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/geonode/Chart.yaml b/deployment/geonode/Chart.yaml index 00c375a..36f7df3 100644 --- a/deployment/geonode/Chart.yaml +++ b/deployment/geonode/Chart.yaml @@ -7,11 +7,11 @@ sources: - https://github.com/zalf-rdm/geonode-k8s dependencies: - name: postgres-operator-ui - version: ~1.10.0 + version: ~1.9.0 repository: https://opensource.zalando.com/postgres-operator/charts/postgres-operator-ui/ condition: postgres-operator-ui.enabled - name: postgres-operator - version: ~1.10.0 + version: ~1.9.0 repository: https://opensource.zalando.com/postgres-operator/charts/postgres-operator/ condition: postgres-operator.enabled - name: rabbitmq From 97c1f2a1131105c8e0bca9721e4f4ce515601e66 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 1 Aug 2023 11:53:27 +0200 Subject: [PATCH 19/31] Align postgres_manifest configuration --- .../geonode/templates/postgres/geonode-manifest.yaml | 6 +++--- deployment/geonode/values.yaml | 2 +- minikube-values.yaml | 9 +++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/deployment/geonode/templates/postgres/geonode-manifest.yaml b/deployment/geonode/templates/postgres/geonode-manifest.yaml index c1c9f84..d92f9dd 100644 --- a/deployment/geonode/templates/postgres/geonode-manifest.yaml +++ b/deployment/geonode/templates/postgres/geonode-manifest.yaml @@ -5,8 +5,8 @@ metadata: spec: teamId: {{ .Release.Name | quote }} volume: - size: {{ .Values.postgres.postgres_operator_manifest.storageSize }} - numberOfInstances: {{ int .Values.postgres.postgres_operator_manifest.numberOfInstances }} + size: {{ .Values.postgres.operator_manifest.storageSize }} + numberOfInstances: {{ int .Values.postgres.operator_manifest.numberOfInstances }} users: {{ .Values.postgres.username }}: - superuser @@ -35,4 +35,4 @@ spec: pg_partman: {{ .Values.postgres.schema }} postgis: {{ .Values.postgres.schema }} postgresql: - version: {{ .Values.postgres_operator_manifest.postgres_version | quote }} + version: {{ .Values.postgres.operator_manifest.postgres_version | quote }} diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index 88f72ba..b2278b7 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -552,7 +552,7 @@ postgres: # get password after creation via: kubectl get secret {{ .Release.name }}.{{ .Release.name }}-{{ container_name }}.credentials -o 'jsonpath={.data.password}' | base64 -d # -- configuration for postgres operator database manifest - postgres_operator_manifest: + operator_manifest: # -- Database storage size storageSize: 3Gi # -- number of database instances diff --git a/minikube-values.yaml b/minikube-values.yaml index 03bac8a..9e69dfa 100644 --- a/minikube-values.yaml +++ b/minikube-values.yaml @@ -40,10 +40,11 @@ postgres-operator-ui: postgres-operator: enabled: True -postgres_operator_manifest: - storageSize: 2Gi - numberOfInstances: 1 - postgres_version: 15 +postgres: + operator_manifest: + storageSize: 2Gi + numberOfInstances: 1 + postgres_version: 15 rabbitmq: replicaCount: 1 From 3032ca29a7472a9baf98f3e53adfc5e328003c50 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 1 Aug 2023 12:56:08 +0200 Subject: [PATCH 20/31] Update helm docs --- deployment/geonode/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/deployment/geonode/README.md b/deployment/geonode/README.md index d99f6ce..0d6da7d 100644 --- a/deployment/geonode/README.md +++ b/deployment/geonode/README.md @@ -16,13 +16,16 @@ Helm Chart for Geonode |------------|------|---------| | https://charts.bitnami.com/bitnami | memcached | ~6.x.x | | https://charts.bitnami.com/bitnami | rabbitmq | ~10.1.7 | -| https://opensource.zalando.com/postgres-operator/charts/postgres-operator-ui/ | postgres-operator-ui | 1.9.0 | -| https://opensource.zalando.com/postgres-operator/charts/postgres-operator/ | postgres-operator | 1.9.0 | +| https://opensource.zalando.com/postgres-operator/charts/postgres-operator-ui/ | postgres-operator-ui | ~1.9.0 | +| https://opensource.zalando.com/postgres-operator/charts/postgres-operator/ | postgres-operator | ~1.9.0 | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| +| geonode.acme.email | string | `"support@example.com"` | the email to be used to gain certificates | +| geonode.acme.enabled | bool | `false` | enables cert-manager to do ACME challenges (aka certificates via letsencrypt) | +| geonode.acme.stageUrl | string | `"https://acme-staging-v02.api.letsencrypt.org/directory"` | ACME staging environment (use acme-staging to avoid running into rate limits) stageUrl: https://acme-v02.api.letsencrypt.org/directory | | geonode.celery.container_name | string | `"celery"` | | | geonode.celery.resources.limits.cpu | int | `1` | limit cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | | geonode.celery.resources.limits.memory | string | `"1Gi"` | limits memory as in resource.limits.memory (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | @@ -61,7 +64,7 @@ Helm Chart for Geonode | geonode.ingress.externalPort | int | `80` | external ingress port | | geonode.ingress.externalScheme | string | `"http"` | external ingress schema. if set to https ingress tls is used. Loading tls certificate via tls-secret options Available options: (http|https) | | geonode.ingress.ingressClassName | string | `nil` | define kubernetes ingress class for geonode ingress | -| geonode.ingress.tlsSecret | string | `"geonode-tls-secret"` | tls certificate for geonode ingress https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/. is used when geonode.ingress.externalScheme is set to https | +| geonode.ingress.tlsSecret | string | `"geonode-tls-secret"` | tls certificate for geonode ingress https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ (for the use of cert-manager, configure the acme section properly). is used when geonode.ingress.externalScheme is set to https | | geonode.ldap.always_update_user | bool | `true` | always update local user database from ldap | | geonode.ldap.attr_map_email_addr | string | `"mailPrimaryAddress"` | email attribute used from ldap | | geonode.ldap.attr_map_first_name | string | `"givenName"` | given name attribute used from ldap | @@ -161,8 +164,7 @@ Helm Chart for Geonode | nginx.resources.requests.cpu | string | `"500m"` | requested cpu as in resource.requests.cpu (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | | nginx.resources.requests.memory | string | `"1Gi"` | requested memory as in resource.requests.memory (https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | | postgres-operator-ui | object | `{"enabled":false,"envs":{"operatorApiUrl":"http://{{ $.Release.Name }}-postgres-operator:8080"},"ingress":{"enabled":false,"hosts":[{"host":"postgres-ui","paths":[""]}],"ingressClassName":null},"replicaCount":1,"service":{"port":80,"type":"ClusterIP"}}` | VALUES DEFINITION: https://github.com/zalando/postgres-operator/blob/master/charts/postgres-operator-ui/values.yaml | -| postgres-operator.api_port | int | `8080` | REST API listener listens to this port | -| postgres-operator.configLoggingRestApi | string | `nil` | | +| postgres-operator.configLoggingRestApi.api_port | int | `8080` | REST API listener listens to this port | | postgres-operator.enabled | bool | `true` | enable postgres-operator (this or postgresql.enabled NOT both ) | | postgres-operator.operatorApiUrl | string | `"http://{{ .Release.Name }}-postgres-operator:8080"` | ??? | | postgres-operator.podServiceAccount | object | `{"name":""}` | not setting the podServiceAccount name will leed to generation of this name. This allows to run multiple postgres-operators in a single kubernetes cluster. just seperating them by namespace. | From ced337afba8acc3aa70c8075f58397665cb55337 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Wed, 2 Aug 2023 11:16:09 +0200 Subject: [PATCH 21/31] Use configured service port The service port is not configurable. --- deployment/geonode/templates/nginx/nginx-ingress.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/geonode/templates/nginx/nginx-ingress.yaml b/deployment/geonode/templates/nginx/nginx-ingress.yaml index be87592..641c3bb 100644 --- a/deployment/geonode/templates/nginx/nginx-ingress.yaml +++ b/deployment/geonode/templates/nginx/nginx-ingress.yaml @@ -31,8 +31,8 @@ spec: backend: service: name: "{{ include "nginx_pod_name" . }}" - port: - number: {{ .Values.geonode.ingress.externalPort }} + port: + number: 80 --- @@ -56,4 +56,4 @@ spec: http01: ingress: ingressClassName: {{ .Values.geonode.ingress.ingressClassName }} -{{ end }} \ No newline at end of file +{{ end }} From da66ecf55e69427c869e63230eca04510234f45a Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Wed, 2 Aug 2023 15:39:50 +0200 Subject: [PATCH 22/31] Temporary overloading entry point Closes #37 --- .../geonode/templates/geoserver/geoserver-deploy.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deployment/geonode/templates/geoserver/geoserver-deploy.yaml b/deployment/geonode/templates/geoserver/geoserver-deploy.yaml index 1ca46cd..1042e33 100644 --- a/deployment/geonode/templates/geoserver/geoserver-deploy.yaml +++ b/deployment/geonode/templates/geoserver/geoserver-deploy.yaml @@ -50,6 +50,13 @@ spec: containers: - name: {{ .Values.geoserver.container_name }} image: "{{ .Values.geoserver.image.name }}:{{ .Values.geoserver.image.tag }}" + # temporary overloading entry point to fix j2 template: https://github.com/GeoNode/geonode/issues/11318 + command: + - sh + - -c + - | + {{`sed -i "s/db:5432/{{DATABASE_HOST}}:5432/g" /templates/geofence/geofence-datasource-ovr.properties.j2`}} + /usr/local/tomcat/tmp/entrypoint.sh ports: - containerPort: {{ .Values.geoserver.port }} From 3a77fefa347e0491c70de936ff330474d5b49c84 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Wed, 2 Aug 2023 15:43:43 +0200 Subject: [PATCH 23/31] added imagePullPolicy, improved logging for geonode containers --- .../templates/geonode/geonode-deploy.yaml | 27 +++++++++++++------ deployment/geonode/values.yaml | 3 ++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/deployment/geonode/templates/geonode/geonode-deploy.yaml b/deployment/geonode/templates/geonode/geonode-deploy.yaml index 2eeb719..c8fd169 100644 --- a/deployment/geonode/templates/geonode/geonode-deploy.yaml +++ b/deployment/geonode/templates/geonode/geonode-deploy.yaml @@ -34,16 +34,18 @@ spec: - tcp://{{ include "rabbit_host" .}} containers: - # This is the django app server + ############## + # django app # + ############## - name: {{ .Values.geonode.container_name }} image: "{{ .Values.geonode.image.name }}:{{ .Values.geonode.image.tag }}" - + imagePullPolicy: {{ .Values.geonode.image.pullPolicy }} command: - bash - -c - | # install dockerize... - wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + wget -q https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz @@ -127,6 +129,13 @@ spec: subPath: geonode-k8s-settings.py readOnly: true + livenessProbe: + httpGet: + path: / + port: 8001 + initialDelaySeconds: 30 + periodSeconds: 5 + resources: requests: memory: {{ .Values.geonode.resources.requests.memory }} @@ -135,16 +144,18 @@ spec: memory: {{ .Values.geonode.resources.limits.memory }} cpu: {{ .Values.geonode.resources.limits.cpu }} - # Celery is the task worker + ########## + # Celery # + ########## - name: {{ .Values.geonode.celery.container_name }} image: "{{ .Values.geonode.image.name }}:{{ .Values.geonode.image.tag }}" - + imagePullPolicy: {{ .Values.geonode.image.pullPolicy }} command: - bash - -c - | # install dockerize... - wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + wget -q https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz @@ -164,8 +175,8 @@ spec: cat /usr/src/geonode/geonode/geonode-k8s-settings.py >> /usr/src/geonode/geonode/settings.py # Setup - touch /usr/src/geonode/invoke.log - dockerize -stdout /usr/src/geonode/invoke.log /usr/src/geonode/entrypoint.sh celery-cmd + touch /var/log/celery.log + dockerize -stdout /var/log/celery.log /usr/src/geonode/entrypoint.sh celery-cmd envFrom: - configMapRef: diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index b2278b7..8a5a3af 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -23,7 +23,8 @@ geonode: name: mwall2bitflow/geonode # -- tag of used geonode image tag: '4.1.x' - + # -- geonode image pull policy + pullPolicy: ifNotPresent # -- additions to tasks.py init script, must be additional code written in python tasks_pre_script: | print("tasks_pre_script not defined ...") From e730fe0a299de74780180ed7357e92d373b42179 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Mon, 31 Jul 2023 14:43:45 +0200 Subject: [PATCH 24/31] prefix variable name of operator manifest --- deployment/geonode/templates/postgres/geonode-manifest.yaml | 4 ++-- deployment/geonode/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/geonode/templates/postgres/geonode-manifest.yaml b/deployment/geonode/templates/postgres/geonode-manifest.yaml index d92f9dd..de8699f 100644 --- a/deployment/geonode/templates/postgres/geonode-manifest.yaml +++ b/deployment/geonode/templates/postgres/geonode-manifest.yaml @@ -5,8 +5,8 @@ metadata: spec: teamId: {{ .Release.Name | quote }} volume: - size: {{ .Values.postgres.operator_manifest.storageSize }} - numberOfInstances: {{ int .Values.postgres.operator_manifest.numberOfInstances }} + size: {{ .Values.postgres.postgres_operator_manifest.storageSize }} + numberOfInstances: {{ int .Values.postgres.postgres_operator_manifest.numberOfInstances }} users: {{ .Values.postgres.username }}: - superuser diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index 8a5a3af..934ff24 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -553,7 +553,7 @@ postgres: # get password after creation via: kubectl get secret {{ .Release.name }}.{{ .Release.name }}-{{ container_name }}.credentials -o 'jsonpath={.data.password}' | base64 -d # -- configuration for postgres operator database manifest - operator_manifest: + postgres_operator_manifest: # -- Database storage size storageSize: 3Gi # -- number of database instances From 80cc43fa9b9d8d565020808692e91d44ab122ff0 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 1 Aug 2023 11:53:27 +0200 Subject: [PATCH 25/31] Align postgres_manifest configuration --- deployment/geonode/templates/postgres/geonode-manifest.yaml | 4 ++-- deployment/geonode/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/geonode/templates/postgres/geonode-manifest.yaml b/deployment/geonode/templates/postgres/geonode-manifest.yaml index de8699f..d92f9dd 100644 --- a/deployment/geonode/templates/postgres/geonode-manifest.yaml +++ b/deployment/geonode/templates/postgres/geonode-manifest.yaml @@ -5,8 +5,8 @@ metadata: spec: teamId: {{ .Release.Name | quote }} volume: - size: {{ .Values.postgres.postgres_operator_manifest.storageSize }} - numberOfInstances: {{ int .Values.postgres.postgres_operator_manifest.numberOfInstances }} + size: {{ .Values.postgres.operator_manifest.storageSize }} + numberOfInstances: {{ int .Values.postgres.operator_manifest.numberOfInstances }} users: {{ .Values.postgres.username }}: - superuser diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index 934ff24..8a5a3af 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -553,7 +553,7 @@ postgres: # get password after creation via: kubectl get secret {{ .Release.name }}.{{ .Release.name }}-{{ container_name }}.credentials -o 'jsonpath={.data.password}' | base64 -d # -- configuration for postgres operator database manifest - postgres_operator_manifest: + operator_manifest: # -- Database storage size storageSize: 3Gi # -- number of database instances From 22a02af554566cb222e7494e3cd9fdfe1236be3d Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Thu, 3 Aug 2023 11:10:00 +0200 Subject: [PATCH 26/31] prefix variable name of operator manifest --- deployment/geonode/templates/postgres/geonode-manifest.yaml | 4 ++-- deployment/geonode/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/geonode/templates/postgres/geonode-manifest.yaml b/deployment/geonode/templates/postgres/geonode-manifest.yaml index d92f9dd..de8699f 100644 --- a/deployment/geonode/templates/postgres/geonode-manifest.yaml +++ b/deployment/geonode/templates/postgres/geonode-manifest.yaml @@ -5,8 +5,8 @@ metadata: spec: teamId: {{ .Release.Name | quote }} volume: - size: {{ .Values.postgres.operator_manifest.storageSize }} - numberOfInstances: {{ int .Values.postgres.operator_manifest.numberOfInstances }} + size: {{ .Values.postgres.postgres_operator_manifest.storageSize }} + numberOfInstances: {{ int .Values.postgres.postgres_operator_manifest.numberOfInstances }} users: {{ .Values.postgres.username }}: - superuser diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index 8a5a3af..c3f3d89 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -24,7 +24,7 @@ geonode: # -- tag of used geonode image tag: '4.1.x' # -- geonode image pull policy - pullPolicy: ifNotPresent + pullPolicy: IfNotPresent # -- additions to tasks.py init script, must be additional code written in python tasks_pre_script: | print("tasks_pre_script not defined ...") From bb3d80c08e61bd6f1f1413970c857331265cb13a Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 1 Aug 2023 11:53:27 +0200 Subject: [PATCH 27/31] Align postgres_manifest configuration --- deployment/geonode/templates/postgres/geonode-manifest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/geonode/templates/postgres/geonode-manifest.yaml b/deployment/geonode/templates/postgres/geonode-manifest.yaml index de8699f..d92f9dd 100644 --- a/deployment/geonode/templates/postgres/geonode-manifest.yaml +++ b/deployment/geonode/templates/postgres/geonode-manifest.yaml @@ -5,8 +5,8 @@ metadata: spec: teamId: {{ .Release.Name | quote }} volume: - size: {{ .Values.postgres.postgres_operator_manifest.storageSize }} - numberOfInstances: {{ int .Values.postgres.postgres_operator_manifest.numberOfInstances }} + size: {{ .Values.postgres.operator_manifest.storageSize }} + numberOfInstances: {{ int .Values.postgres.operator_manifest.numberOfInstances }} users: {{ .Values.postgres.username }}: - superuser From 9d534e397491314d4a457ec37ba74f82b97150cb Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Thu, 3 Aug 2023 10:58:09 +0200 Subject: [PATCH 28/31] minor changes --- deployment/geonode/templates/nginx/nginx-conf.yaml | 6 ++++++ deployment/geonode/templates/pycsw/pycsw-deploy.yaml | 2 -- deployment/geonode/values.yaml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/deployment/geonode/templates/nginx/nginx-conf.yaml b/deployment/geonode/templates/nginx/nginx-conf.yaml index cf65dec..95333b2 100644 --- a/deployment/geonode/templates/nginx/nginx-conf.yaml +++ b/deployment/geonode/templates/nginx/nginx-conf.yaml @@ -115,6 +115,12 @@ data: location {{ .Values.pycsw.endpoint }} { client_max_body_size {{ .Values.nginx.maxClientBodySize }}; + gzip_static always; + expires 30d; + access_log off; + add_header Pragma "public"; + add_header Cache-Control "max-age=31536000, public"; + # compression gzip on; gzip_types diff --git a/deployment/geonode/templates/pycsw/pycsw-deploy.yaml b/deployment/geonode/templates/pycsw/pycsw-deploy.yaml index df7a9f6..06bf1c6 100644 --- a/deployment/geonode/templates/pycsw/pycsw-deploy.yaml +++ b/deployment/geonode/templates/pycsw/pycsw-deploy.yaml @@ -10,8 +10,6 @@ spec: matchLabels: org.geonode.instance: "{{ include "pycsw_pod_name" . }}" serviceName: "{{ include "pycsw_pod_name" . }}" - strategy: - type: Recreate template: metadata: labels: diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index c3f3d89..567c5fe 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -369,7 +369,7 @@ pycsw: # -- pycsw docker image name: geopython/pycsw # -- pycsw docker image tag - tag: '2.6.0' + tag: '2.6.1' # -- pycsw endpoint port port: 8000 # -- pycsw url below geonode.ingress.externalDomain From fdca5a62858cbf3672ed9d0ed9acd9a761068990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20Wallschl=C3=A4ger?= Date: Tue, 5 Sep 2023 08:03:34 +0200 Subject: [PATCH 29/31] Update pycsw.md --- docs/pycsw.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pycsw.md b/docs/pycsw.md index dbc028e..42f6f37 100644 --- a/docs/pycsw.md +++ b/docs/pycsw.md @@ -1,7 +1,7 @@ External pyCSW -------------- -GeoNode-k8s external pycsw container. This allows changes to pyCSW with having to change GeoNode codebase and allows scalability of the pyCSW service seperatly. The external pycsw service is by default enabled. To disable set: `.Values.pycsw.enabled = False` +GeoNode-k8s external pycsw container. This allows changes to pyCSW without having to change GeoNode codebase and allows scalability of the pyCSW service seperatly. The external pycsw service is by default enabled. To disable set: `.Values.pycsw.enabled = False` The pycsw configuration (pycsw.cfg) and mappings are defined in the values file: `[ .Values.pycsw.config , .Values.pycsw.mappings ]` From e1ac75236cffcfe11c47f9428a55d75110513961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20Wallschl=C3=A4ger?= Date: Tue, 5 Sep 2023 08:05:08 +0200 Subject: [PATCH 30/31] Update pycsw-cfg.yaml --- deployment/geonode/templates/pycsw/pycsw-cfg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/geonode/templates/pycsw/pycsw-cfg.yaml b/deployment/geonode/templates/pycsw/pycsw-cfg.yaml index 6d17a03..5ed5df0 100644 --- a/deployment/geonode/templates/pycsw/pycsw-cfg.yaml +++ b/deployment/geonode/templates/pycsw/pycsw-cfg.yaml @@ -67,4 +67,4 @@ data: {{ if .Values.pycsw.config.inspire.contact_name }} contact_name={{ .Values.pycsw.config.inspire.contact_name }}{{ end }} {{ if .Values.pycsw.config.inspire.contact_email }} contact_email={{ .Values.pycsw.config.inspire.contact_email }}{{ end }} {{ if .Values.pycsw.config.inspire.temp_extent }} temp_extent={{ .Values.pycsw.config.inspire.temp_extent }}{{ end }} -{{ end }} \ No newline at end of file +{{ end }} From 36bfa7ad190dbcc1c998451a635f7759a2f74044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20Wallschl=C3=A4ger?= Date: Tue, 5 Sep 2023 08:10:14 +0200 Subject: [PATCH 31/31] Update pycsw-mappings-py.yaml --- deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml b/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml index e9fb666..aeff0be 100644 --- a/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml +++ b/deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml @@ -6,4 +6,4 @@ metadata: namespace: {{ .Release.Namespace }} data: pycsw-mappings.py: {{- .Values.pycsw.mappings | toYaml | indent 1 }} -{{ end }} \ No newline at end of file +{{ end }}