diff --git a/docs/en/connector-v2/source/Jira.md b/docs/en/connector-v2/source/Jira.md new file mode 100644 index 00000000000..0f170b695d8 --- /dev/null +++ b/docs/en/connector-v2/source/Jira.md @@ -0,0 +1,160 @@ +# Jira + +> Jira source connector + +## Description + +Used to read data from Jira. + +## Key features + +- [x] [batch](../../concept/connector-v2-features.md) +- [ ] [stream](../../concept/connector-v2-features.md) +- [ ] [exactly-once](../../concept/connector-v2-features.md) +- [x] [schema projection](../../concept/connector-v2-features.md) +- [ ] [parallelism](../../concept/connector-v2-features.md) +- [ ] [support user-defined split](../../concept/connector-v2-features.md) + +## Options + +| name | type | required | default value | +| --------------------------- | ------ | -------- | ------------- | +| url | String | Yes | - | +| email | String | Yes | - | +| api_token | String | Yes | - | +| method | String | No | get | +| schema.fields | Config | No | - | +| format | String | No | json | +| params | Map | No | - | +| body | String | No | - | +| poll_interval_ms | int | No | - | +| retry | int | No | - | +| retry_backoff_multiplier_ms | int | No | 100 | +| retry_backoff_max_ms | int | No | 10000 | +| common-options | config | No | - | + +### url [String] + +http request url + +### email [String] + +Jira Email + +### api_token [String] + +Jira API Token + +https://id.atlassian.com/manage-profile/security/api-tokens + +### method [String] + +http request method, only supports GET, POST method + +### params [Map] + +http params + +### body [String] + +http body + +### poll_interval_ms [int] + +request http api interval(millis) in stream mode + +### retry [int] + +The max retry times if request http return to `IOException` + +### retry_backoff_multiplier_ms [int] + +The retry-backoff times(millis) multiplier if request http failed + +### retry_backoff_max_ms [int] + +The maximum retry-backoff times(millis) if request http failed + +### format [String] + +the format of upstream data, now only support `json` `text`, default `json`. + +when you assign format is `json`, you should also assign schema option, for example: + +upstream data is the following: + +```json + +{"code": 200, "data": "get success", "success": true} + +``` + +you should assign schema as the following: + +```hocon + +schema { + fields { + code = int + data = string + success = boolean + } +} + +``` + +connector will generate data as the following: + +| code | data | success | +|------|-------------|---------| +| 200 | get success | true | + +when you assign format is `text`, connector will do nothing for upstream data, for example: + +upstream data is the following: + +```json + +{"code": 200, "data": "get success", "success": true} + +``` + +connector will generate data as the following: + +| content | +|---------| +| {"code": 200, "data": "get success", "success": true} | + +### schema [Config] + +#### fields [Config] + +the schema fields of upstream data + +### common options + +Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details + +## Example + +```hocon +Jira { + url = "https://liugddx.atlassian.net/rest/api/3/search" + email = "test@test.com" + api_token = "xxx" + schema { + fields { + expand = string + startAt = bigint + maxResults = int + total = int + } + } +} +``` + +## Changelog + +### next version + +- Add Jira Source Connector diff --git a/plugin-mapping.properties b/plugin-mapping.properties index 913c3fc6b82..e42a472de3a 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -152,4 +152,5 @@ seatunnel.source.Lemlist = connector-http-lemlist seatunnel.source.Klaviyo = connector-http-klaviyo seatunnel.sink.Slack = connector-slack seatunnel.source.OneSignal = connector-http-onesignal +seatunnel.source.Jira = connector-http-jira seatunnel.source.Gitlab = connector-http-gitlab diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/pom.xml b/seatunnel-connectors-v2/connector-http/connector-http-jira/pom.xml new file mode 100644 index 00000000000..34b7830ead6 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/pom.xml @@ -0,0 +1,40 @@ + + + + + connector-http + org.apache.seatunnel + ${revision} + + 4.0.0 + + connector-http-jira + + + + org.apache.seatunnel + connector-http-base + ${project.version} + + + + diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java new file mode 100644 index 00000000000..6d112de4c75 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jira.source; + +import static org.apache.seatunnel.connectors.seatunnel.http.util.AuthorizationUtil.getTokenByBasicAuth; + +import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.config.CheckConfigUtil; +import org.apache.seatunnel.common.config.CheckResult; +import org.apache.seatunnel.common.constants.JobMode; +import org.apache.seatunnel.common.constants.PluginType; +import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader; +import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext; +import org.apache.seatunnel.connectors.seatunnel.http.source.HttpSource; +import org.apache.seatunnel.connectors.seatunnel.http.source.HttpSourceReader; +import org.apache.seatunnel.connectors.seatunnel.jira.source.config.JiraSourceConfig; +import org.apache.seatunnel.connectors.seatunnel.jira.source.config.JiraSourceParameter; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@AutoService(SeaTunnelSource.class) +public class JiraSource extends HttpSource { + private final JiraSourceParameter jiraSourceParameter = new JiraSourceParameter(); + + @Override + public String getPluginName() { + return "Jira"; + } + + @Override + public Boundedness getBoundedness() { + if (JobMode.BATCH.equals(jobContext.getJobMode())) { + return Boundedness.BOUNDED; + } + throw new UnsupportedOperationException("Jira source connector not support unbounded operation"); + } + + @Override + public void prepare(Config pluginConfig) throws PrepareFailException { + CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, JiraSourceConfig.URL.key(), JiraSourceConfig.EMAIL.key(), JiraSourceConfig.API_TOKEN.key()); + if (!result.isSuccess()) { + throw new PrepareFailException(getPluginName(), PluginType.SOURCE, result.getMsg()); + } + //get accessToken by basic auth + String accessToken = getTokenByBasicAuth(pluginConfig.getString(JiraSourceConfig.EMAIL.key()), pluginConfig.getString(JiraSourceConfig.API_TOKEN.key())); + jiraSourceParameter.buildWithConfig(pluginConfig, accessToken); + buildSchemaWithConfig(pluginConfig); + } + + @Override + public AbstractSingleSplitReader createReader(SingleSplitReaderContext readerContext) throws Exception { + return new HttpSourceReader(this.jiraSourceParameter, readerContext, this.deserializationSchema); + } +} diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java new file mode 100644 index 00000000000..c37f949b5d3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jira.source; + +import org.apache.seatunnel.api.configuration.util.Condition; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.api.table.factory.TableSourceFactory; +import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig; +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpRequestMethod; +import org.apache.seatunnel.connectors.seatunnel.jira.source.config.JiraSourceConfig; + +import com.google.auto.service.AutoService; + +@AutoService(Factory.class) +public class JiraSourceFactory implements TableSourceFactory { + @Override + public String factoryIdentifier() { + return "Jira"; + } + + @Override + public OptionRule optionRule() { + return OptionRule.builder() + .required(JiraSourceConfig.URL) + .required(JiraSourceConfig.EMAIL) + .required(JiraSourceConfig.API_TOKEN) + .optional(JiraSourceConfig.METHOD) + .optional(JiraSourceConfig.HEADERS) + .optional(JiraSourceConfig.PARAMS) + .conditional(Condition.of(HttpConfig.METHOD, HttpRequestMethod.POST), JiraSourceConfig.BODY) + .conditional(Condition.of(HttpConfig.FORMAT, "json"), SeaTunnelSchema.SCHEMA) + .optional(JiraSourceConfig.FORMAT) + .optional(JiraSourceConfig.POLL_INTERVAL_MILLS) + .optional(JiraSourceConfig.RETRY) + .optional(JiraSourceConfig.RETRY_BACKOFF_MAX_MS) + .optional(JiraSourceConfig.RETRY_BACKOFF_MULTIPLIER_MS) + .build(); + } +} diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java new file mode 100644 index 00000000000..9eda2e6e458 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jira.source.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig; + +public class JiraSourceConfig extends HttpConfig { + public static final String AUTHORIZATION = "Authorization"; + public static final Option EMAIL = Options.key("email") + .stringType() + .noDefaultValue() + .withDescription("Jira email"); + + public static final Option API_TOKEN = Options.key("api_token") + .stringType() + .noDefaultValue() + .withDescription("Jira API Token"); +} diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceParameter.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceParameter.java new file mode 100644 index 00000000000..3e1ae6dbd57 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceParameter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jira.source.config; + +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpParameter; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import java.util.HashMap; + +public class JiraSourceParameter extends HttpParameter { + public void buildWithConfig(Config pluginConfig, String accessToken) { + super.buildWithConfig(pluginConfig); + // put authorization in headers + this.headers = this.getHeaders() == null ? new HashMap<>() : this.getHeaders(); + this.headers.put(JiraSourceConfig.AUTHORIZATION, accessToken); + this.setHeaders(this.headers); + } +} diff --git a/seatunnel-connectors-v2/connector-http/pom.xml b/seatunnel-connectors-v2/connector-http/pom.xml index e0de4766c08..ce24c0fc920 100644 --- a/seatunnel-connectors-v2/connector-http/pom.xml +++ b/seatunnel-connectors-v2/connector-http/pom.xml @@ -37,6 +37,7 @@ connector-http-lemlist connector-http-klaviyo connector-http-onesignal + connector-http-jira connector-http-gitlab diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index 8aad7aad0f6..6e2d1803323 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -363,6 +363,12 @@ ${project.version} provided + + org.apache.seatunnel + connector-http-jira + ${project.version} + provided + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml index d3fd0e8469d..ed1cf6bd81f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml @@ -68,6 +68,12 @@ ${project.version} test + + org.apache.seatunnel + connector-http-jira + ${project.version} + test + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java new file mode 100644 index 00000000000..94a513e171c --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.connector.http; + +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.TestContainer; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.util.stream.Stream; + +public class HttpJiraIT extends TestSuiteBase implements TestResource { + + private static final String IMAGE = "mockserver/mockserver:5.14.0"; + + private GenericContainer mockserverContainer; + + @BeforeAll + @Override + public void startUp() { + this.mockserverContainer = new GenericContainer<>(DockerImageName.parse(IMAGE)) + .withNetwork(NETWORK) + .withNetworkAliases("mockserver") + .withExposedPorts(1080) + .withCopyFileToContainer(MountableFile.forHostPath(new File(HttpJiraIT.class.getResource( + "/mockserver-jira-config.json").getPath()).getAbsolutePath()), + "/tmp/mockserver-jira-config.json") + .withEnv("MOCKSERVER_INITIALIZATION_JSON_PATH", "/tmp/mockserver-jira-config.json") + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) + .waitingFor(new HttpWaitStrategy().forPath("/").forStatusCode(404)); + Startables.deepStart(Stream.of(mockserverContainer)).join(); + } + + @AfterAll + @Override + public void tearDown() { + if (mockserverContainer != null) { + mockserverContainer.stop(); + } + } + + @TestTemplate + public void testHttpJiraSourceToAssertSink(TestContainer container) throws IOException, InterruptedException { + Container.ExecResult execResult = container.executeJob("/jira_json_to_assert.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf new file mode 100644 index 00000000000..3be4e859870 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + Jira { + url = "http://mockserver:1080/rest/api/3/search" + email = "admin@test.com" + api_token = "token" + method = "GET" + format = "json" + schema = { + fields { + expand = string + startAt = int + maxResults = int + total = string + } + } + } +} + +sink { + Console {} + Assert { + rules { + field_rules = [ + { + field_name = expand + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = startAt + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = maxResults + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json new file mode 100644 index 00000000000..0072f7f9320 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json @@ -0,0 +1,20 @@ +// https://www.mock-server.com/mock_server/getting_started.html#request_matchers + +[ + { + "httpRequest": { + "method" : "GET", + "path": "/rest/api/3/search" + }, + "httpResponse": { + "body": [ + { + "expand": "schema,names", + "startAt": 0, + "maxResults": 50, + "total": 3 + } + ] + } + } +]