From d1f84fedd0ca37ce87b4e8a84329c9d45ffe708e Mon Sep 17 00:00:00 2001 From: haifxu Date: Fri, 1 Mar 2024 12:55:37 +0800 Subject: [PATCH] [INLONG-9750][Dashboard] Add Kafka source for Ingestion (#9751) --- .../src/plugins/sources/defaults/Kafka.ts | 118 ++++++++++++++++++ .../src/plugins/sources/defaults/index.ts | 6 + inlong-dashboard/src/ui/locales/cn.json | 4 + inlong-dashboard/src/ui/locales/en.json | 4 + .../ui/pages/ModuleAudit/IpModule/config.tsx | 4 +- 5 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 inlong-dashboard/src/plugins/sources/defaults/Kafka.ts diff --git a/inlong-dashboard/src/plugins/sources/defaults/Kafka.ts b/inlong-dashboard/src/plugins/sources/defaults/Kafka.ts new file mode 100644 index 0000000000..da0ff9d0b1 --- /dev/null +++ b/inlong-dashboard/src/plugins/sources/defaults/Kafka.ts @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { DataWithBackend } from '@/plugins/DataWithBackend'; +import { RenderRow } from '@/plugins/RenderRow'; +import { RenderList } from '@/plugins/RenderList'; +import { SourceInfo } from '../common/SourceInfo'; +import i18n from '@/i18n'; + +const { I18n } = DataWithBackend; +const { FieldDecorator, SyncField, IngestionField } = RenderRow; +const { ColumnDecorator } = RenderList; + +export default class KafkaSource + extends SourceInfo + implements DataWithBackend, RenderRow, RenderList +{ + @FieldDecorator({ + type: 'input', + rules: [{ required: true }], + props: values => ({ + disabled: values?.status === 101, + placeholder: 'localhost:9092', + }), + }) + @SyncField() + @IngestionField() + @ColumnDecorator() + @I18n('meta.Sources.Kafka.BootstrapServers') + bootstrapServers: string; + + @FieldDecorator({ + type: 'input', + rules: [{ required: true }], + props: values => ({ + disabled: values?.status === 101, + }), + }) + @SyncField() + @IngestionField() + @ColumnDecorator() + @I18n('Topic') + topic: string; + + @FieldDecorator({ + type: 'select', + initialValue: 'Latest', + rules: [{ required: true }], + props: values => ({ + disabled: values?.status === 101, + options: [ + { + label: 'Earliest', + value: 'earliest', + }, + { + label: 'Latest', + value: 'latest', + }, + { + label: 'None', + value: 'none', + }, + ], + }), + }) + @IngestionField() + @I18n('meta.Sources.Kafka.autoOffsetReset') + autoOffsetReset: string; + + @FieldDecorator({ + type: 'input', + tooltip: i18n.t('meta.Sources.Kafka.partitionOffsetsHelp'), + props: values => ({ + disabled: values?.status === 101, + placeholder: '0#0_1#0_2#0', + }), + }) + @SyncField() + @IngestionField() + @ColumnDecorator() + @I18n('meta.Sources.Kafka.partitionOffsets') + partitionOffsets: string; + + @FieldDecorator({ + type: 'select', + initialValue: 'GMT+8:00', + props: values => ({ + disabled: Boolean(values.id), + options: [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, + -12, + ].map(item => ({ + label: Math.sign(item) === 1 || Math.sign(item) === 0 ? `GMT+${item}:00` : `GMT${item}:00`, + value: Math.sign(item) === 1 || Math.sign(item) === 0 ? `GMT+${item}:00` : `GMT${item}:00`, + })), + }), + }) + @IngestionField() + @I18n('meta.Sources.File.TimeZone') + dataTimeZone: string; +} diff --git a/inlong-dashboard/src/plugins/sources/defaults/index.ts b/inlong-dashboard/src/plugins/sources/defaults/index.ts index aea16927cb..58d55b30cb 100644 --- a/inlong-dashboard/src/plugins/sources/defaults/index.ts +++ b/inlong-dashboard/src/plugins/sources/defaults/index.ts @@ -39,6 +39,12 @@ export const allDefaultSources: MetaExportWithBackendList = [ useSync: false, LoadEntity: () => import('./File'), }, + { + label: 'Kafka', + value: 'KAFKA', + useSync: false, + LoadEntity: () => import('./Kafka'), + }, { label: 'MySQL', value: 'MYSQL_BINLOG', diff --git a/inlong-dashboard/src/ui/locales/cn.json b/inlong-dashboard/src/ui/locales/cn.json index 15dc20f325..1d6d5a035f 100644 --- a/inlong-dashboard/src/ui/locales/cn.json +++ b/inlong-dashboard/src/ui/locales/cn.json @@ -137,6 +137,10 @@ "meta.Sources.Pulsar.DataEscapeChar": "数据转义符", "meta.Sources.Pulsar.WrapType": "消息打包格式", "meta.Sources.Pulsar.SerializationType": "数据格式", + "meta.Sources.Kafka.BootstrapServers": "Bootstrap Servers", + "meta.Sources.Kafka.autoOffsetReset": "自动偏移重置", + "meta.Sources.Kafka.partitionOffsets": "分区位点", + "meta.Sources.Kafka.partitionOffsetsHelp": "自定义各分区消费位点，格式为: {分区}#{位点}，并使用下划线(_)连接，如：0#0_1#3_2#1。不指定则从 topic 已提交的 offset 开始消费", "meta.Sinks.SinkName": "名称", "meta.Sinks.SinkNameRule": "只能包含英文字母、数字、点号(.)、中划线(-)、下划线(_)", "meta.Sinks.SinkType": "类型", diff --git a/inlong-dashboard/src/ui/locales/en.json b/inlong-dashboard/src/ui/locales/en.json index 94de245a1a..9b27256e86 100644 --- a/inlong-dashboard/src/ui/locales/en.json +++ b/inlong-dashboard/src/ui/locales/en.json @@ -137,6 +137,10 @@ "meta.Sources.Pulsar.DataEscapeChar": "Data escape char", "meta.Sources.Pulsar.WrapType": "Wrap type", "meta.Sources.Pulsar.SerializationType": "Data type", + "meta.Sources.Kafka.BootstrapServers": "Bootstrap Servers", + "meta.Sources.Kafka.autoOffsetReset": "Auto Offset Reset", + "meta.Sources.Kafka.partitionOffsets": "Partition Offsets", + "meta.Sources.Kafka.partitionOffsetsHelp": "Customize the offsets of each partition, the format is: {partition}#{offsets}, and use underscores (_) to connect, such as: 0#0_1#3_2#1. If not specified, consumption will start from the submitted offsets of the topic.", "meta.Sinks.SinkName": "Name", "meta.Sinks.SinkNameRule": "Only English letters, numbers, dots(.), minus(-), and underscores(_)", "meta.Sinks.SinkType": "Type", diff --git a/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx b/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx index 0e9676457f..0351ed9088 100644 --- a/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx +++ b/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx @@ -68,7 +68,7 @@ export const getFormContent = (initialValues, onSearch) => [ props: { allowClear: false, showTime: true, - format: 'YYYY-MM-DD HH:MM:ss', + format: 'YYYY-MM-DD HH:mm:ss', }, }, { @@ -79,7 +79,7 @@ export const getFormContent = (initialValues, onSearch) => [ props: { allowClear: false, showTime: true, - format: 'YYYY-MM-DD HH:MM:ss', + format: 'YYYY-MM-DD HH:mm:ss', }, }, {