env: CRD manifests

config/config.d/01-clickhouse-05-trace_log.xml

@@ -4,11 +4,11 @@
 <!-- Edit appropriate template in the following folder: -->
 <!-- deploy/builder/templates-config -->
 <!-- IMPORTANT -->
-<yandex>
-    <trace_log replace="1">
-        <database>system</database>
-        <table>trace_log</table>
-        <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
-        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
-    </trace_log>
-</yandex>
+<yandex>
+    <trace_log replace="1">
+        <database>system</database>
+        <table>trace_log</table>
+        <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </trace_log>
+</yandex>

deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml

@@ -1100,19 +1100,19 @@ spec:
                                 topologyKey:
                                   type: string
                                   description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity"
-                          spec:
-                            # TODO specify PodSpec
-                            type: object
-                            description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details"
-                            # nullable: true
-                            x-kubernetes-preserve-unknown-fields: true
                           metadata:
                             type: object
                             description: |
                               allows pass standard object's metadata from template to Pod
                               More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
                             # nullable: true
                             x-kubernetes-preserve-unknown-fields: true
+                          spec:
+                            # TODO specify PodSpec
+                            type: object
+                            description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details"
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
 
                     volumeClaimTemplates:
                       type: array

deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml

@@ -64,6 +64,18 @@ spec:
               description: |
                 Current ClickHouseKeeperInstallation status, contains many fields like overall status, desired replicas and ready replica list with their endpoints
               properties:
+                chop-version:
+                  type: string
+                  description: "ClickHouse operator version"
+                chop-commit:
+                  type: string
+                  description: "ClickHouse operator git commit SHA"
+                chop-date:
+                  type: string
+                  description: "ClickHouse operator build date"
+                chop-ip:
+                  type: string
+                  description: "IP address of the operator's pod which managed this CHI"
                 status:
                   type: string
                   description: "Status"
@@ -88,16 +100,24 @@ spec:
                       secure:
                         type: string
                         description: if a secure connection to Keeper is required
+                normalized:
+                  type: object
+                  description: "Normalized CHK requested"
+                  x-kubernetes-preserve-unknown-fields: true
+                normalizedCompleted:
+                  type: object
+                  description: "Normalized CHK completed"
+                  x-kubernetes-preserve-unknown-fields: true
             spec:
               type: object
               description: KeeperSpec defines the desired state of a Keeper cluster
               properties:
-                type:
+                namespaceDomainPattern:
                   type: string
-                  description: This is used by chi reconciliation.
-                  enum:
-                    - "Container"
-                    - "Embedded"
+                  description: |
+                    Custom domain pattern which will be used for DNS names of `Service` or `Pod`.
+                    Typical use scenario - custom cluster domain in Kubernetes cluster
+                    Example: %s.svc.my.test
                 replicas:
                   type: integer
                   format: int32
@@ -106,41 +126,139 @@ spec:
                     The valid range of size is from 1 to 7.
                   minimum: 1
                   maximum: 7
-                settings:
-                  type: object
-                  description: "allows configure multiple aspects and behavior for `clickhouse-keeper` instance"
-                  x-kubernetes-preserve-unknown-fields: true
-                podTemplate:
+                configuration:
                   type: object
-                  description: |
-                    podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone
+                  description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource"
+                  # nullable: true
                   properties:
-                    metadata:
-                      type: object
-                      description: |
-                        allows passing standard object's metadata from template to Pod
-                        More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
-                      x-kubernetes-preserve-unknown-fields: true
-                    spec:
-                      # TODO specify PodSpec
+                    settings:
                       type: object
-                      description: "allows defining whole Pod.spec inside StatefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details"
+                      description: "allows configure multiple aspects and behavior for `clickhouse-keeper` instance"
                       x-kubernetes-preserve-unknown-fields: true
-                volumeClaimTemplates:
-                  type: array
-                  description: "allows defining template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mounting clickhouse `data`, clickhouse `logs` or something else"
-                  items:
-                    type: object
-                    properties:
-                      metadata:
+                    clusters:
+                      type: array
+                      description: |
+                        describes ClickHouseKeeper clusters layout and allows change settings on cluster-level and replica-level
+                      # nullable: true
+                      items:
                         type: object
-                        description: |
-                          allows to pass standard object's metadata from template to PVC
-                          More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
-                        x-kubernetes-preserve-unknown-fields: true
-                      spec:
+                        #required:
+                        #  - name
+                        properties:
+                          name:
+                            type: string
+                            description: "cluster name, used to identify set of ClickHouseKeeper servers and wide used during generate names of related Kubernetes resources"
+                            minLength: 1
+                            # See namePartClusterMaxLen const
+                            maxLength: 15
+                            pattern: "^[a-zA-Z0-9-]{0,15}$"
+                          layout:
+                            type: object
+                            description: |
+                              describe current cluster layout, how many replicas
+                            # nullable: true
+                            properties:
+                              replicasCount:
+                                type: integer
+                                description: "how many replicas in ClickHouseKeeper cluster"
+                templates:
+                  type: object
+                  description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it"
+                  # nullable: true
+                  properties:
+                    podTemplates:
+                      type: array
+                      description: |
+                        podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone
+                        More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates
+                      # nullable: true
+                      items:
+                        type: object
+                        #required:
+                        #  - name
+                        properties:
+                          name:
+                            type: string
+                            description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`"
+                          metadata:
+                            type: object
+                            description: |
+                              allows pass standard object's metadata from template to Pod
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+                          spec:
+                            # TODO specify PodSpec
+                            type: object
+                            description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details"
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+
+                    volumeClaimTemplates:
+                      type: array
+                      description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else"
+                      # nullable: true
+                      items:
+                        type: object
+                        #required:
+                        #  - name
+                        #  - spec
+                        properties:
+                          name:
+                            type: string
+                            description: |
+                              template name, could use to link inside
+                              top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`,
+                              cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`,
+                              shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate`
+                              replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate`
+                          metadata:
+                            type: object
+                            description: |
+                              allows to pass standard object's metadata from template to PVC
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+                          spec:
+                            type: object
+                            description: |
+                              allows define all aspects of `PVC` resource
+                              More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+                    serviceTemplates:
+                      type: array
+                      description: |
+                        allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level
+                      # nullable: true
+                      items:
                         type: object
-                        description: |
-                          allows define all aspects of `PVC` resource
-                          More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
-                        x-kubernetes-preserve-unknown-fields: true
+                        #required:
+                        #  - name
+                        #  - spec
+                        properties:
+                          name:
+                            type: string
+                            description: |
+                              template name, could use to link inside
+                              chi-level `chi.spec.defaults.templates.serviceTemplate`
+                              cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate`
+                              shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate`
+                              replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate`
+                          metadata:
+                            # TODO specify ObjectMeta
+                            type: object
+                            description: |
+                              allows pass standard object's metadata from template to Service
+                              Could be use for define specificly for Cloud Provider metadata which impact to behavior of service
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
+                          spec:
+                            # TODO specify ServiceSpec
+                            type: object
+                            description: |
+                              describe behavior of generated Service
+                              More info: https://kubernetes.io/docs/concepts/services-networking/service/
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true

deploy/helm/README.md

@@ -49,7 +49,7 @@ For upgrade please install CRDs separately:
 | operator.image.repository | string | `"altinity/clickhouse-operator"` | image repository |
 | operator.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) |
 | operator.resources | object | `{}` | custom resource configuration, look `kubectl explain pod.spec.containers.resources` for details |
-| podAnnotations | object | `{"prometheus.io/port":"8888","prometheus.io/scrape":"true"}` | annotations to add to the clickhouse-operator pod, look `kubectl explain pod.spec.annotations` for details |
+| podAnnotations | object | `{"clickhouse-operator-metrics/port":"9999","clickhouse-operator-metrics/scrape":"true","prometheus.io/port":"8888","prometheus.io/scrape":"true"}` | annotations to add to the clickhouse-operator pod, look `kubectl explain pod.spec.annotations` for details |
 | podSecurityContext | object | `{}` |  |
 | secret.create | bool | `true` | create a secret with operator credentials |
 | secret.password | string | `"clickhouse_operator_password"` | operator credentials password |
@@ -61,5 +61,3 @@ For upgrade please install CRDs separately:
 | serviceMonitor.enabled | bool | `false` | ServiceMonitor Custom resource is created for a (prometheus-operator)[https://github.com/prometheus-operator/prometheus-operator] |
 | tolerations | list | `[]` | tolerations for scheduler pod assignment, look `kubectl explain pod.spec.tolerations` for details |
 
-----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.11.3](https://github.com/norwoodj/helm-docs/releases/v1.11.3)

deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml

@@ -1095,19 +1095,19 @@ spec:
                                 topologyKey:
                                   type: string
                                   description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity"
-                          spec:
-                            # TODO specify PodSpec
-                            type: object
-                            description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details"
-                            # nullable: true
-                            x-kubernetes-preserve-unknown-fields: true
                           metadata:
                             type: object
                             description: |
                               allows pass standard object's metadata from template to Pod
                               More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
                             # nullable: true
                             x-kubernetes-preserve-unknown-fields: true
+                          spec:
+                            # TODO specify PodSpec
+                            type: object
+                            description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details"
+                            # nullable: true
+                            x-kubernetes-preserve-unknown-fields: true
                     volumeClaimTemplates:
                       type: array
                       description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else"