Skip to content

Commit 4464129

Browse files
BryanCutlersabbasaniSthuthiGhosh9400lithinwxdsteveburnett
authored
Adding Arrow Flight connector template (prestodb#24427)
This adds the Arrow Flight base module as a template to build connectors that use a Flight service to transfer data with Presto in Arrow format. A concrete Arrow Flight connector implementation will extend the BaseArrowFlightClientHandler that can connect to a Flight service that handles the required RPC calls from the client handler. An example connector implementation is provided in the unit testing. RFC: https://github.com/prestodb/rfcs/blob/main/RFC-0004-arrow-flight-connector.md Co-authored-by: sai bhaskar reddy <sai.bhaskar.reddy.sabbasani1@ibm.com> Co-authored-by: SthuthiGhosh9400 <Sthuthi.Ghosh@ibm.com> Co-authored-by: lithinwxd <Lithin.Purushothaman@ibm.com> Co-authored-by: Steve Burnett <burnett@pobox.com> Co-authored-by: elbinpallimalilibm <elbin.pallimalil@ibm.com> Co-authored-by: Tim Meehan <tim@timdmeehan.com>
1 parent c229714 commit 4464129

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+6343
-1
lines changed

‎.github/workflows/arrow-flight-tests.yml

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: arrow flight tests
2+
3+
on:
4+
pull_request:
5+
6+
env:
7+
CONTINUOUS_INTEGRATION: true
8+
MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError"
9+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
10+
MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true"
11+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end"
12+
RETRY: .github/bin/retry
13+
14+
jobs:
15+
changes:
16+
runs-on: ubuntu-latest
17+
permissions:
18+
pull-requests: read
19+
outputs:
20+
codechange: ${{ steps.filter.outputs.codechange }}
21+
steps:
22+
- uses: dorny/paths-filter@v2
23+
id: filter
24+
with:
25+
filters: |
26+
codechange:
27+
- '!presto-docs/**'
28+
test:
29+
runs-on: ubuntu-latest
30+
needs: changes
31+
strategy:
32+
fail-fast: false
33+
matrix:
34+
modules:
35+
- ":presto-base-arrow-flight" # Only run tests for the `presto-base-arrow-flight` module
36+
37+
timeout-minutes: 80
38+
concurrency:
39+
group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }}
40+
cancel-in-progress: true
41+
42+
steps:
43+
# Checkout the code only if there are changes in the relevant files
44+
- uses: actions/checkout@v4
45+
if: needs.changes.outputs.codechange == 'true'
46+
with:
47+
show-progress: false
48+
49+
# Set up Java for the build environment
50+
- uses: actions/setup-java@v2
51+
if: needs.changes.outputs.codechange == 'true'
52+
with:
53+
distribution: 'temurin'
54+
java-version: 8
55+
56+
# Cache Maven dependencies to speed up the build
57+
- name: Cache local Maven repository
58+
if: needs.changes.outputs.codechange == 'true'
59+
id: cache-maven
60+
uses: actions/cache@v2
61+
with:
62+
path: ~/.m2/repository
63+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
64+
restore-keys: |
65+
${{ runner.os }}-maven-2-
66+
67+
# Resolve Maven dependencies (if cache is not found)
68+
- name: Populate Maven cache
69+
if: steps.cache-maven.outputs.cache-hit != 'true' && needs.changes.outputs.codechange == 'true'
70+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs
71+
72+
# Install dependencies for the target module
73+
- name: Maven Install
74+
if: needs.changes.outputs.codechange == 'true'
75+
run: |
76+
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
77+
./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }}
78+
79+
# Run Maven tests for the target module
80+
- name: Maven Tests
81+
if: needs.changes.outputs.codechange == 'true'
82+
run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }}

‎.github/workflows/test-other-modules.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -87,4 +87,5 @@ jobs:
8787
!presto-test-coverage,
8888
!presto-iceberg,
8989
!presto-singlestore,
90-
!presto-native-sidecar-plugin'
90+
!presto-native-sidecar-plugin,
91+
!presto-base-arrow-flight'

‎pom.xml

+26
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
<dep.gson.version>2.11.0</dep.gson.version>
9090
<dep.commons.lang3.version>3.14.0</dep.commons.lang3.version>
9191
<dep.guice.version>5.1.0</dep.guice.version>
92+
<dep.arrow.version>17.0.0</dep.arrow.version>
9293

9394
<!--
9495
America/Bahia_Banderas has:
@@ -209,6 +210,7 @@
209210
<module>presto-hana</module>
210211
<module>presto-openapi</module>
211212
<module>presto-native-sidecar-plugin</module>
213+
<module>presto-base-arrow-flight</module>
212214
</modules>
213215

214216
<dependencyManagement>
@@ -2442,6 +2444,30 @@
24422444
<version>2.12.0</version>
24432445
</dependency>
24442446

2447+
<dependency>
2448+
<groupId>org.apache.arrow</groupId>
2449+
<artifactId>arrow-memory-core</artifactId>
2450+
<version>${dep.arrow.version}</version>
2451+
</dependency>
2452+
2453+
<dependency>
2454+
<groupId>org.apache.arrow</groupId>
2455+
<artifactId>arrow-memory-netty</artifactId>
2456+
<version>${dep.arrow.version}</version>
2457+
</dependency>
2458+
2459+
<dependency>
2460+
<groupId>org.apache.arrow</groupId>
2461+
<artifactId>arrow-vector</artifactId>
2462+
<version>${dep.arrow.version}</version>
2463+
</dependency>
2464+
2465+
<dependency>
2466+
<groupId>org.apache.arrow</groupId>
2467+
<artifactId>flight-core</artifactId>
2468+
<version>${dep.arrow.version}</version>
2469+
</dependency>
2470+
24452471
</dependencies>
24462472
</dependencyManagement>
24472473

‎presto-base-arrow-flight/pom.xml

+256
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
<?xml version="1.0"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<parent>
6+
<groupId>com.facebook.presto</groupId>
7+
<artifactId>presto-root</artifactId>
8+
<version>0.292-SNAPSHOT</version>
9+
</parent>
10+
11+
<artifactId>presto-base-arrow-flight</artifactId>
12+
<name>presto-base-arrow-flight</name>
13+
<description>Presto - Base Arrow Flight Connector</description>
14+
15+
<properties>
16+
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
17+
</properties>
18+
19+
<dependencies>
20+
<dependency>
21+
<groupId>org.apache.arrow</groupId>
22+
<artifactId>arrow-memory-core</artifactId>
23+
<exclusions>
24+
<exclusion>
25+
<groupId>org.slf4j</groupId>
26+
<artifactId>slf4j-api</artifactId>
27+
</exclusion>
28+
</exclusions>
29+
</dependency>
30+
31+
<dependency>
32+
<groupId>org.apache.arrow</groupId>
33+
<artifactId>arrow-memory-netty</artifactId>
34+
<scope>runtime</scope>
35+
<exclusions>
36+
<exclusion>
37+
<groupId>org.slf4j</groupId>
38+
<artifactId>slf4j-api</artifactId>
39+
</exclusion>
40+
</exclusions>
41+
</dependency>
42+
43+
<dependency>
44+
<groupId>org.apache.arrow</groupId>
45+
<artifactId>arrow-vector</artifactId>
46+
<exclusions>
47+
<exclusion>
48+
<groupId>org.slf4j</groupId>
49+
<artifactId>slf4j-api</artifactId>
50+
</exclusion>
51+
<!-- Excluding to resolve upper bound errors -->
52+
<exclusion>
53+
<groupId>com.fasterxml.jackson.datatype</groupId>
54+
<artifactId>jackson-datatype-jsr310</artifactId>
55+
</exclusion>
56+
</exclusions>
57+
</dependency>
58+
59+
<dependency>
60+
<groupId>org.apache.arrow</groupId>
61+
<artifactId>flight-core</artifactId>
62+
<exclusions>
63+
<exclusion>
64+
<groupId>org.slf4j</groupId>
65+
<artifactId>slf4j-api</artifactId>
66+
</exclusion>
67+
</exclusions>
68+
</dependency>
69+
70+
<dependency>
71+
<groupId>com.facebook.airlift</groupId>
72+
<artifactId>bootstrap</artifactId>
73+
</dependency>
74+
75+
<dependency>
76+
<groupId>com.facebook.airlift</groupId>
77+
<artifactId>log</artifactId>
78+
</dependency>
79+
80+
<dependency>
81+
<groupId>com.google.guava</groupId>
82+
<artifactId>guava</artifactId>
83+
</dependency>
84+
85+
<dependency>
86+
<groupId>javax.inject</groupId>
87+
<artifactId>javax.inject</artifactId>
88+
</dependency>
89+
90+
<dependency>
91+
<groupId>com.facebook.presto</groupId>
92+
<artifactId>presto-spi</artifactId>
93+
</dependency>
94+
95+
<dependency>
96+
<groupId>io.airlift</groupId>
97+
<artifactId>slice</artifactId>
98+
</dependency>
99+
100+
<dependency>
101+
<groupId>com.facebook.airlift</groupId>
102+
<artifactId>log-manager</artifactId>
103+
</dependency>
104+
105+
<dependency>
106+
<groupId>com.fasterxml.jackson.core</groupId>
107+
<artifactId>jackson-annotations</artifactId>
108+
</dependency>
109+
110+
<dependency>
111+
<groupId>com.facebook.presto</groupId>
112+
<artifactId>presto-common</artifactId>
113+
</dependency>
114+
115+
<dependency>
116+
<groupId>com.fasterxml.jackson.core</groupId>
117+
<artifactId>jackson-databind</artifactId>
118+
</dependency>
119+
120+
<dependency>
121+
<groupId>com.google.code.findbugs</groupId>
122+
<artifactId>jsr305</artifactId>
123+
<optional>true</optional>
124+
</dependency>
125+
126+
<dependency>
127+
<groupId>com.google.inject</groupId>
128+
<artifactId>guice</artifactId>
129+
</dependency>
130+
131+
<dependency>
132+
<groupId>com.facebook.airlift</groupId>
133+
<artifactId>configuration</artifactId>
134+
</dependency>
135+
136+
<dependency>
137+
<groupId>joda-time</groupId>
138+
<artifactId>joda-time</artifactId>
139+
</dependency>
140+
141+
<dependency>
142+
<groupId>org.jdbi</groupId>
143+
<artifactId>jdbi3-core</artifactId>
144+
</dependency>
145+
146+
<!-- for testing -->
147+
<dependency>
148+
<groupId>org.testng</groupId>
149+
<artifactId>testng</artifactId>
150+
<scope>test</scope>
151+
</dependency>
152+
153+
<dependency>
154+
<groupId>io.airlift.tpch</groupId>
155+
<artifactId>tpch</artifactId>
156+
<scope>test</scope>
157+
</dependency>
158+
159+
<dependency>
160+
<groupId>com.facebook.presto</groupId>
161+
<artifactId>presto-tpch</artifactId>
162+
<scope>test</scope>
163+
</dependency>
164+
165+
<dependency>
166+
<groupId>com.facebook.airlift</groupId>
167+
<artifactId>json</artifactId>
168+
<scope>test</scope>
169+
</dependency>
170+
171+
<dependency>
172+
<groupId>com.facebook.presto</groupId>
173+
<artifactId>presto-testng-services</artifactId>
174+
<scope>test</scope>
175+
</dependency>
176+
177+
<dependency>
178+
<groupId>com.facebook.airlift</groupId>
179+
<artifactId>testing</artifactId>
180+
<scope>test</scope>
181+
</dependency>
182+
183+
<dependency>
184+
<groupId>com.facebook.presto</groupId>
185+
<artifactId>presto-main</artifactId>
186+
<scope>test</scope>
187+
</dependency>
188+
189+
<dependency>
190+
<groupId>com.facebook.presto</groupId>
191+
<artifactId>presto-tests</artifactId>
192+
<scope>test</scope>
193+
</dependency>
194+
195+
<dependency>
196+
<groupId>com.h2database</groupId>
197+
<artifactId>h2</artifactId>
198+
<scope>test</scope>
199+
</dependency>
200+
201+
<dependency>
202+
<groupId>org.apache.arrow</groupId>
203+
<artifactId>arrow-jdbc</artifactId>
204+
<version>${dep.arrow.version}</version>
205+
<scope>test</scope>
206+
<exclusions>
207+
<exclusion>
208+
<groupId>org.slf4j</groupId>
209+
<artifactId>slf4j-api</artifactId>
210+
</exclusion>
211+
</exclusions>
212+
</dependency>
213+
</dependencies>
214+
215+
<build>
216+
<plugins>
217+
<plugin>
218+
<groupId>org.apache.maven.plugins</groupId>
219+
<artifactId>maven-surefire-plugin</artifactId>
220+
<configuration>
221+
<argLine>-Xss10M</argLine>
222+
</configuration>
223+
</plugin>
224+
<plugin>
225+
<groupId>org.apache.maven.plugins</groupId>
226+
<artifactId>maven-enforcer-plugin</artifactId>
227+
</plugin>
228+
<plugin>
229+
<groupId>org.apache.maven.plugins</groupId>
230+
<artifactId>maven-dependency-plugin</artifactId>
231+
</plugin>
232+
<plugin>
233+
<groupId>org.basepom.maven</groupId>
234+
<artifactId>duplicate-finder-maven-plugin</artifactId>
235+
<version>1.2.1</version>
236+
<configuration>
237+
<ignoredClassPatterns>
238+
<ignoredClassPattern>module-info</ignoredClassPattern>
239+
<ignoredClassPattern>META-INF.versions.9.module-info</ignoredClassPattern>
240+
</ignoredClassPatterns>
241+
<ignoredResourcePatterns>
242+
<ignoredResourcePattern>arrow-git.properties</ignoredResourcePattern>
243+
<ignoredResourcePattern>about.html</ignoredResourcePattern>
244+
</ignoredResourcePatterns>
245+
</configuration>
246+
<executions>
247+
<execution>
248+
<goals>
249+
<goal>check</goal>
250+
</goals>
251+
</execution>
252+
</executions>
253+
</plugin>
254+
</plugins>
255+
</build>
256+
</project>

0 commit comments

Comments
 (0)