-
Notifications
You must be signed in to change notification settings - Fork 148
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Chen Dai <daichen@amazon.com>
- Loading branch information
Showing
3 changed files
with
76 additions
and
0 deletions.
There are no files selected for viewing
27 changes: 27 additions & 0 deletions
27
...a/org/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGenerator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.sql.planner.streaming.watermark; | ||
|
||
import lombok.RequiredArgsConstructor; | ||
|
||
/** | ||
* Watermark generator that generates watermark with bounded out-of-order latency. | ||
*/ | ||
@RequiredArgsConstructor | ||
public class BoundedOutOfOrderWatermarkGenerator implements WatermarkGenerator { | ||
|
||
/** The maximum out-of-order allowed. */ | ||
private final long maxOutOfOrderAllowed; | ||
|
||
/** The maximum timestamp seen so far. */ | ||
private long maxTimestamp; | ||
|
||
@Override | ||
public long generate(long timestamp) { | ||
maxTimestamp = Math.max(maxTimestamp, timestamp); | ||
return (maxTimestamp - maxOutOfOrderAllowed - 1); | ||
} | ||
} |
21 changes: 21 additions & 0 deletions
21
core/src/main/java/org/opensearch/sql/planner/streaming/watermark/WatermarkGenerator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.sql.planner.streaming.watermark; | ||
|
||
/** | ||
* A watermark generator generates watermark timestamp based on some strategy. | ||
*/ | ||
public interface WatermarkGenerator { | ||
|
||
/** | ||
* Generate watermark timestamp on the given event timestamp. | ||
* | ||
* @param timestamp event timestamp. | ||
* @return watermark timestamp | ||
*/ | ||
long generate(long timestamp); | ||
|
||
} |
28 changes: 28 additions & 0 deletions
28
...g/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGeneratorTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.sql.planner.streaming.watermark; | ||
|
||
import static org.junit.jupiter.api.Assertions.assertEquals; | ||
|
||
import org.junit.jupiter.api.Test; | ||
|
||
class BoundedOutOfOrderWatermarkGeneratorTest { | ||
|
||
@Test | ||
void shouldAdvanceWatermarkIfLaterEvent() { | ||
BoundedOutOfOrderWatermarkGenerator generator = new BoundedOutOfOrderWatermarkGenerator(100); | ||
assertEquals(899, generator.generate(1000)); | ||
assertEquals(1899, generator.generate(2000)); | ||
} | ||
|
||
@Test | ||
void shouldNotChangeWatermarkByLateEvent() { | ||
BoundedOutOfOrderWatermarkGenerator generator = new BoundedOutOfOrderWatermarkGenerator(100); | ||
assertEquals(899, generator.generate(1000)); | ||
assertEquals(899, generator.generate(500)); | ||
assertEquals(899, generator.generate(700)); | ||
} | ||
} |