Issue#27 pycsw single microservice

deployment/geonode/README.md

@@ -23,6 +23,7 @@ Helm Chart for Geonode
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+
 | favicon | string | AAABAAMAEBAAAAEAIABoBA ... AAAA== | A base64 encoded favicon |
 | geonode.acme.email | string | `"support@example.com"` | the email to be used to gain certificates |
 | geonode.acme.enabled | bool | `false` | enables cert-manager to do ACME challenges (aka certificates via letsencrypt) |

deployment/geonode/templates/_helpers.tpl

@@ -17,6 +17,10 @@
 {{ .Release.Name }}-{{ .Values.nginx.pod_name }}
 {{- end -}}
 
+{{- define "pycsw_pod_name" -}}
+{{ .Release.Name }}-{{ .Values.pycsw.pod_name }}
+{{- end -}}
+
 
 
 # Database definitions

deployment/geonode/templates/geonode/geonode-deploy.yaml

@@ -49,10 +49,12 @@ spec:
         - tcp://{{ include "rabbit_host" .}}
 
       containers:
-      # This is the django app server
+      ##############
+      # django app #
+      ##############
       - name: {{ .Values.geonode.container_name }}
         image: "{{ .Values.geonode.image.name }}:{{ .Values.geonode.image.tag }}"
-
+        imagePullPolicy: {{ .Values.geonode.image.pullPolicy }}
         command:
         - bash
         - -c
@@ -147,6 +149,13 @@ spec:
           readOnly: true
         {{ end }}
 
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8001
+          initialDelaySeconds: 30
+          periodSeconds: 5
+
         resources:
           requests:
             memory: {{ .Values.geonode.resources.requests.memory }}
@@ -155,10 +164,12 @@ spec:
             memory: {{ .Values.geonode.resources.limits.memory }}
             cpu: {{ .Values.geonode.resources.limits.cpu }}
 
-      # Celery is the task worker
+      ##########
+      # Celery #
+      ##########
       - name: {{ .Values.geonode.celery.container_name }}
         image: "{{ .Values.geonode.image.name }}:{{ .Values.geonode.image.tag }}"
-
+        imagePullPolicy: {{ .Values.geonode.image.pullPolicy }}
         command:
         - bash
         - -c

deployment/geonode/templates/geonode/geonode-env.yaml

@@ -127,6 +127,13 @@ data:
   LDAP_USER_ATTR_MAP_LAST_NAME: {{ .Values.geonode.ldap.attr_map_last_name | quote }}
   LDAP_USER_ATTR_MAP_EMAIL_ADDR: {{ .Values.geonode.ldap.attr_map_email_addr | quote }}
 
+  # Configure PYCSW
+  {{ if .Values.pycsw.enabled }}
+  CATALOGUE_ENGINE: geonode.catalogue.backends.pycsw_http
+  CATALOGUE_URL: "http://{{ include "pycsw_pod_name" . }}:{{ .Values.pycsw.port }}"
+  {{ else }}
+  CATALOGUE_ENGINE: geonode.catalogue.backends.pycsw_local
+  {{ end}}
   # OAuth2
   # TODO (mwall) implement OAUTH2
   OAUTH2_API_KEY: ""

deployment/geonode/templates/nginx/nginx-conf.yaml

@@ -110,6 +110,34 @@ data:
         proxy_http_version 1.1;
       }
 
+      {{ if .Values.pycsw.enabled }}
+      # external PYCSW forward 
+      location {{ .Values.pycsw.endpoint }} {
+        client_max_body_size {{ .Values.nginx.maxClientBodySize }};
+
+        gzip_static always;
+        expires 30d;
+        access_log off;
+        add_header Pragma "public";
+        add_header Cache-Control "max-age=31536000, public";
+
+        # compression
+        gzip on;
+        gzip_types
+          text/xml
+          text/plain
+          application/xml
+          application/xml+rss
+          application/json;
+
+        proxy_pass http://{{ include "pycsw_pod_name" . }}:{{ .Values.pycsw.port }};
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+      }
+      {{ end }}
+
       location / {
         client_max_body_size {{ .Values.nginx.maxClientBodySize }};
 

deployment/geonode/templates/pycsw/pycsw-cfg.yaml

@@ -0,0 +1,70 @@
+{{ if .Values.pycsw.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-pycsw-cfg
+  namespace: {{ .Release.Namespace }}
+data:
+  pycsw.cfg: |+
+    [server]
+    {{ if .Values.pycsw.config.server.home }}    home={{ .Values.pycsw.config.server.home }}{{ end }}
+        url={{ .Values.geonode.ingress.externalScheme}}://{{ .Values.geonode.ingress.externalDomain}}{{ .Values.pycsw.endpoint }}
+    {{ if .Values.pycsw.config.server.mimetype }}    mimetype={{ .Values.pycsw.config.server.mimetype }}{{ end }}
+    {{ if .Values.pycsw.config.server.encoding }}    encoding={{ .Values.pycsw.config.server.encoding }}{{ end }}
+    {{ if .Values.pycsw.config.server.language }}    language={{ .Values.pycsw.config.server.language }}{{ end }}
+    {{ if .Values.pycsw.config.server.maxrecords }}    maxrecords={{ .Values.pycsw.config.server.maxrecords }}{{ end }}
+    {{ if .Values.pycsw.config.server.loglevel }}    loglevel={{ .Values.pycsw.config.server.loglevel }}{{ end }}
+    {{ if .Values.pycsw.config.server.logfile }}    logfile={{ .Values.pycsw.config.server.logfile }}{{ end }}
+    {{ if .Values.pycsw.config.server.ogc_schemas_base }}    ogc_schemas_base={{ .Values.pycsw.config.server.ogc_schemas_base }}{{ end }}
+    {{ if .Values.pycsw.config.server.federatedcatalogues }}    federatedcatalogues={{ .Values.pycsw.config.server.federatedcatalogues }}{{ end }}
+    {{ if .Values.pycsw.config.server.pretty_print }}    pretty_print={{ .Values.pycsw.config.server.pretty_print }}{{ end }}
+    {{ if .Values.pycsw.config.server.gzip_compresslevel }}    gzip_compresslevel={{ .Values.pycsw.config.server.gzip_compresslevel }}{{ end }}
+    {{ if .Values.pycsw.config.server.domainquerytype }}    domainquerytype={{ .Values.pycsw.config.server.domainquerytype }}{{ end }}
+    {{ if .Values.pycsw.config.server.domaincounts }}     domaincounts={{ .Values.pycsw.config.server.domaincounts }}{{ end }}
+    {{ if .Values.pycsw.config.server.spatial_ranking }}    spatial_ranking={{ .Values.pycsw.config.server.spatial_ranking }}{{ end }}
+    {{ if .Values.pycsw.config.server.profiles }}    profiles={{ .Values.pycsw.config.server.profiles }}{{ end }}
+    {{ if .Values.pycsw.config.server.workers }}    workers={{ .Values.pycsw.config.server.workers }}{{ end }}
+    {{ if .Values.pycsw.config.server.timeout }}    timeout={{ .Values.pycsw.config.server.timeout }}{{ end }}
+    [manager]
+    {{ if .Values.pycsw.config.manager.transactions }}    transactions={{ .Values.pycsw.config.manager.transactions }}{{ end }}
+    {{ if .Values.pycsw.config.manager.allowed_ips }}    allowed_ips={{ .Values.pycsw.config.manager.allowed_ips }}{{ end }}
+    {{ if .Values.pycsw.config.manager.csw_harvest_pagesize }}    csw_harvest_pagesize={{ .Values.pycsw.config.manager.csw_harvest_pagesize }}{{ end }}
+    [metadata:main]
+    {{ if .Values.pycsw.config.metadata.identification_title }}    identification_title={{ .Values.pycsw.config.metadata.identification_title }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.identification_abstract }}    identification_abstract={{ .Values.pycsw.config.metadata.identification_abstract }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.identification_keywords }}    identification_keywords={{ .Values.pycsw.config.metadata.identification_keywords }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.identification_keywords_type }}    identification_keywords_type={{ .Values.pycsw.config.metadata.identification_keywords_type }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.identification_fees }}    identification_fees={{ .Values.pycsw.config.metadata.identification_fees }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.identification_accessconstraints }}    identification_accessconstraints={{ .Values.pycsw.config.metadata.identification_accessconstraints }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.provider_name }}    provider_name={{ .Values.pycsw.config.metadata.provider_name }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.provider_url }}    provider_url={{ .Values.pycsw.config.metadata.provider_url }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_name }}    contact_name={{ .Values.pycsw.config.metadata.contact_name }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_position }}    contact_position={{ .Values.pycsw.config.metadata.contact_position }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_address }}    contact_address={{ .Values.pycsw.config.metadata.contact_address }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_city }}    contact_city={{ .Values.pycsw.config.metadata.contact_city }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_stateorprovince }}    contact_stateorprovince={{ .Values.pycsw.config.metadata.contact_stateorprovince }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_postalcode }}    contact_postalcode={{ .Values.pycsw.config.metadata.contact_postalcode }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_country }}    contact_country={{ .Values.pycsw.config.metadata.contact_country }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_phone }}    contact_phone={{ .Values.pycsw.config.metadata.contact_phone }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_fax }}    contact_fax={{ .Values.pycsw.config.metadata.contact_fax }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_email }}    contact_email={{ .Values.pycsw.config.metadata.contact_email }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_url }}    contact_url={{ .Values.pycsw.config.metadata.contact_url }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_hours }}    contact_hours={{ .Values.pycsw.config.metadata.contact_hours }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_instructions }}    contact_instructions={{ .Values.pycsw.config.metadata.contact_instructions }}{{ end }}
+    {{ if .Values.pycsw.config.metadata.contact_role }}    contact_role={{ .Values.pycsw.config.metadata.contact_role }}{{ end }}
+    [repository]
+        database=${PYCSW_REPOSITORY_DATABASE_URI}
+    {{ if .Values.pycsw.config.repository.mappings }}    mappings={{ .Values.pycsw.config.repository.mappings }}{{ end }}
+    {{ if .Values.pycsw.config.repository.table }}    table={{ .Values.pycsw.config.repository.table }}{{ end }}
+    {{ if .Values.pycsw.config.repository.filter }}    filter={{ .Values.pycsw.config.repository.filter }}{{ end }}
+    [metadata:inspire]
+    {{ if .Values.pycsw.config.inspire.enabled }}    enabled={{ .Values.pycsw.config.inspire.enabled }}{{ end }}
+    {{ if .Values.pycsw.config.inspire.languages_supported }}    languages_supported={{ .Values.pycsw.config.inspire.languages_supported }}{{ end }}
+    {{ if .Values.pycsw.config.inspire.default_language }}    default_language={{ .Values.pycsw.config.inspire.default_language }}{{ end }}
+    {{ if .Values.pycsw.config.inspire.date }}    date={{ .Values.pycsw.config.inspire.date }}{{ end }}
+    {{ if .Values.pycsw.config.inspire.gemet_keywords }}    gemet_keywords={{ .Values.pycsw.config.inspire.gemet_keywords }}{{ end }}
+    {{ if .Values.pycsw.config.inspire.conformity_service }}    conformity_service={{ .Values.pycsw.config.inspire.conformity_service }}{{ end }}
+    {{ if .Values.pycsw.config.inspire.contact_name }}    contact_name={{ .Values.pycsw.config.inspire.contact_name }}{{ end }}
+    {{ if .Values.pycsw.config.inspire.contact_email }}    contact_email={{ .Values.pycsw.config.inspire.contact_email }}{{ end }}
+    {{ if .Values.pycsw.config.inspire.temp_extent }}    temp_extent={{ .Values.pycsw.config.inspire.temp_extent }}{{ end }}
+{{ end }}

deployment/geonode/templates/pycsw/pycsw-deploy.yaml

@@ -0,0 +1,91 @@
+{{ if .Values.pycsw.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: "{{ include "pycsw_pod_name" . }}"
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: {{ .Values.pycsw.replicaCount }}
+  selector:
+    matchLabels:
+      org.geonode.instance: "{{ include "pycsw_pod_name" . }}"
+  serviceName: "{{ include "pycsw_pod_name" . }}"
+  template:
+    metadata:
+      labels:
+        org.geonode.instance: "{{ include "pycsw_pod_name" . }}"
+      annotations:
+        checksum/pycsw-env: {{ include (print $.Template.BasePath "/pycsw/pycsw-env.yaml") . | sha256sum }}
+        checksum/pycsw-cfg: {{ include (print $.Template.BasePath "/pycsw/pycsw-cfg.yaml") . | sha256sum }}
+        checksum/pycsw-mappings-py: {{ include (print $.Template.BasePath "/pycsw/pycsw-mappings-py.yaml") . | sha256sum }}
+    spec:
+      terminationGracePeriodSeconds: 3
+      initContainers:
+      # Wait for GeoNode to be up and running, else there can be a race conddition where pycsw creates the database table,
+      # and lets crash init process of geonode
+      - name: pycsw-wait-for-geonode
+        image: alpine/curl
+        imagePullPolicy: IfNotPresent
+        command: ["/bin/sh","-c"]
+        args: ['while [ $(curl -ksw "%{http_code}" "$GEONODE_ENDPOINT:8001" -o /dev/null) -ne 200 ]; do sleep 5; echo "health check failed . Waiting for GeoNode ($GEONODE_ENDPOINT:8001) ..."; done']
+        env:
+        - name: GEONODE_ENDPOINT
+          value: "{{ include "geonode_pod_name" . }}"
+
+      containers:
+      - name: {{ .Values.pycsw.container_name }}
+        image: "{{ .Values.pycsw.image.name }}:{{ .Values.pycsw.image.tag }}"
+        envFrom:
+          - configMapRef:
+              name: {{ include "pycsw_pod_name" . }}-env
+        env:
+        - name: PYCSW_SERVER_URL
+          value: "{{ .Values.geonode.ingress.externalDomain}}{{ .Values.pycsw.endpoint }}"
+        - name: GEONODE_DATABASE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name:  {{ .Values.postgres.geonodedatabase }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do
+              key: password
+        - name: PYCSW_REPOSITORY_DATABASE_URI
+          value: "postgresql://$(GEONODE_DATABASE):$(GEONODE_DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(GEONODE_DATABASE)"
+        ports:
+        - containerPort: {{ .Values.pycsw.port }}
+        volumeMounts:
+        - name: pycsw-cfg
+          mountPath: "/etc/pycsw/pycsw.cfg"
+          subPath: pycsw.cfg
+          readOnly: true
+        - name: pycsw-mappings-py
+          mountPath: {{ .Values.pycsw.config.repository.mappings }}
+          subPath: pycsw-mappings.py
+          readOnly: true
+        resources:
+          requests:
+            memory: {{ .Values.pycsw.resources.requests.memory }}
+            cpu: {{ .Values.pycsw.resources.requests.cpu }}
+          limits:
+            memory: {{ .Values.pycsw.resources.limits.memory }}
+            cpu: {{ .Values.pycsw.resources.limits.cpu }}
+        livenessProbe:
+          httpGet:
+            path: /
+            port: {{ .Values.pycsw.port }}
+          initialDelaySeconds: 3
+          periodSeconds: 10
+
+      volumes:
+      - name: pycsw-cfg
+        configMap:
+          name: {{ .Release.Name }}-pycsw-cfg
+          defaultMode: 0744
+          items:
+          - key: pycsw.cfg
+            path: "pycsw.cfg"
+      - name: pycsw-mappings-py
+        configMap:
+          name: {{ .Release.Name }}-pycsw-mappings-py
+          defaultMode: 0744
+          items:
+          - key: pycsw-mappings.py
+            path: "pycsw-mappings.py"
+{{ end }}

deployment/geonode/templates/pycsw/pycsw-env.yaml

@@ -0,0 +1,13 @@
+{{ if .Values.pycsw.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "pycsw_pod_name" . }}-env
+  namespace: {{ .Release.Namespace }}
+data:
+  # Database Settings
+  DATABASE_HOST: "{{ include "postgres_pod_name" . }}"
+  DATABASE_PORT: "{{ include "database_port" .}}"
+  GEONODE_DATABASE: {{ .Values.postgres.geonodedatabase | quote }}
+  GEONODE_DATABASE_SCHEMA: {{ .Values.postgres.schema }}
+{{ end }}

deployment/geonode/templates/pycsw/pycsw-mappings-py.yaml

@@ -0,0 +1,9 @@
+{{ if .Values.pycsw.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-pycsw-mappings-py
+  namespace: {{ .Release.Namespace }}
+data:
+  pycsw-mappings.py: {{- .Values.pycsw.mappings | toYaml | indent 1 }}
+{{ end }}

deployment/geonode/templates/pycsw/pycsw-svc.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.pycsw.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ include "pycsw_pod_name" . }}"
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    org.geonode.instance: "{{ include "pycsw_pod_name" . }}"
+  ports:
+  - targetPort: 8000
+    port: {{ .Values.pycsw.port }}
+    name: pycsw
+{{- end }}