[INLONG-9750][Dashboard] Add Kafka source for Ingestion (#9751)

apache · Mar 1, 2024 · d1f84fe · d1f84fe
1 parent 896bb54
commit d1f84fe
Show file tree

Hide file tree

Showing 5 changed files with 134 additions and 2 deletions.
diff --git a/inlong-dashboard/src/plugins/sources/defaults/Kafka.ts b/inlong-dashboard/src/plugins/sources/defaults/Kafka.ts
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { DataWithBackend } from '@/plugins/DataWithBackend';
+import { RenderRow } from '@/plugins/RenderRow';
+import { RenderList } from '@/plugins/RenderList';
+import { SourceInfo } from '../common/SourceInfo';
+import i18n from '@/i18n';
+
+const { I18n } = DataWithBackend;
+const { FieldDecorator, SyncField, IngestionField } = RenderRow;
+const { ColumnDecorator } = RenderList;
+
+export default class KafkaSource
+  extends SourceInfo
+  implements DataWithBackend, RenderRow, RenderList
+{
+  @FieldDecorator({
+    type: 'input',
+    rules: [{ required: true }],
+    props: values => ({
+      disabled: values?.status === 101,
+      placeholder: 'localhost:9092',
+    }),
+  })
+  @SyncField()
+  @IngestionField()
+  @ColumnDecorator()
+  @I18n('meta.Sources.Kafka.BootstrapServers')
+  bootstrapServers: string;
+
+  @FieldDecorator({
+    type: 'input',
+    rules: [{ required: true }],
+    props: values => ({
+      disabled: values?.status === 101,
+    }),
+  })
+  @SyncField()
+  @IngestionField()
+  @ColumnDecorator()
+  @I18n('Topic')
+  topic: string;
+
+  @FieldDecorator({
+    type: 'select',
+    initialValue: 'Latest',
+    rules: [{ required: true }],
+    props: values => ({
+      disabled: values?.status === 101,
+      options: [
+        {
+          label: 'Earliest',
+          value: 'earliest',
+        },
+        {
+          label: 'Latest',
+          value: 'latest',
+        },
+        {
+          label: 'None',
+          value: 'none',
+        },
+      ],
+    }),
+  })
+  @IngestionField()
+  @I18n('meta.Sources.Kafka.autoOffsetReset')
+  autoOffsetReset: string;
+
+  @FieldDecorator({
+    type: 'input',
+    tooltip: i18n.t('meta.Sources.Kafka.partitionOffsetsHelp'),
+    props: values => ({
+      disabled: values?.status === 101,
+      placeholder: '0#0_1#0_2#0',
+    }),
+  })
+  @SyncField()
+  @IngestionField()
+  @ColumnDecorator()
+  @I18n('meta.Sources.Kafka.partitionOffsets')
+  partitionOffsets: string;
+
+  @FieldDecorator({
+    type: 'select',
+    initialValue: 'GMT+8:00',
+    props: values => ({
+      disabled: Boolean(values.id),
+      options: [
+        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11,
+        -12,
+      ].map(item => ({
+        label: Math.sign(item) === 1 || Math.sign(item) === 0 ? `GMT+${item}:00` : `GMT${item}:00`,
+        value: Math.sign(item) === 1 || Math.sign(item) === 0 ? `GMT+${item}:00` : `GMT${item}:00`,
+      })),
+    }),
+  })
+  @IngestionField()
+  @I18n('meta.Sources.File.TimeZone')
+  dataTimeZone: string;
+}
diff --git a/inlong-dashboard/src/plugins/sources/defaults/index.ts b/inlong-dashboard/src/plugins/sources/defaults/index.ts
@@ -39,6 +39,12 @@ export const allDefaultSources: MetaExportWithBackendList<SourceMetaType> = [
     useSync: false,
     LoadEntity: () => import('./File'),
   },
+  {
+    label: 'Kafka',
+    value: 'KAFKA',
+    useSync: false,
+    LoadEntity: () => import('./Kafka'),
+  },
   {
     label: 'MySQL',
     value: 'MYSQL_BINLOG',

diff --git a/inlong-dashboard/src/ui/locales/cn.json b/inlong-dashboard/src/ui/locales/cn.json
@@ -137,6 +137,10 @@
   "meta.Sources.Pulsar.DataEscapeChar": "数据转义符",
   "meta.Sources.Pulsar.WrapType": "消息打包格式",
   "meta.Sources.Pulsar.SerializationType": "数据格式",
+  "meta.Sources.Kafka.BootstrapServers": "Bootstrap Servers",
+  "meta.Sources.Kafka.autoOffsetReset": "自动偏移重置",
+  "meta.Sources.Kafka.partitionOffsets": "分区位点",
+  "meta.Sources.Kafka.partitionOffsetsHelp": "自定义各分区消费位点，格式为: {分区}#{位点}，并使用下划线(_)连接，如：0#0_1#3_2#1。不指定则从 topic 已提交的 offset 开始消费",
   "meta.Sinks.SinkName": "名称",
   "meta.Sinks.SinkNameRule": "只能包含英文字母、数字、点号(.)、中划线(-)、下划线(_)",
   "meta.Sinks.SinkType": "类型",

diff --git a/inlong-dashboard/src/ui/locales/en.json b/inlong-dashboard/src/ui/locales/en.json
@@ -137,6 +137,10 @@
   "meta.Sources.Pulsar.DataEscapeChar": "Data escape char",
   "meta.Sources.Pulsar.WrapType": "Wrap type",
   "meta.Sources.Pulsar.SerializationType": "Data type",
+  "meta.Sources.Kafka.BootstrapServers": "Bootstrap Servers",
+  "meta.Sources.Kafka.autoOffsetReset": "Auto Offset Reset",
+  "meta.Sources.Kafka.partitionOffsets": "Partition Offsets",
+  "meta.Sources.Kafka.partitionOffsetsHelp": "Customize the offsets of each partition, the format is: {partition}#{offsets}, and use underscores (_) to connect, such as: 0#0_1#3_2#1. If not specified, consumption will start from the submitted offsets of the topic.",
   "meta.Sinks.SinkName": "Name",
   "meta.Sinks.SinkNameRule": "Only English letters, numbers, dots(.), minus(-), and underscores(_)",
   "meta.Sinks.SinkType": "Type",

diff --git a/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx b/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx
@@ -68,7 +68,7 @@ export const getFormContent = (initialValues, onSearch) => [
     props: {
       allowClear: false,
       showTime: true,
-      format: 'YYYY-MM-DD HH:MM:ss',
+      format: 'YYYY-MM-DD HH:mm:ss',
     },
   },
   {
@@ -79,7 +79,7 @@ export const getFormContent = (initialValues, onSearch) => [
     props: {
       allowClear: false,
       showTime: true,
-      format: 'YYYY-MM-DD HH:MM:ss',
+      format: 'YYYY-MM-DD HH:mm:ss',
     },
   },
   {