forked from apache/iotdb
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' of https://github.com/apache/iotdb into support…
…-delete
- Loading branch information
Showing
237 changed files
with
9,642 additions
and
2,707 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
132 changes: 132 additions & 0 deletions
132
integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.iotdb.db.it; | ||
|
|
||
| import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; | ||
| import org.apache.iotdb.it.env.EnvFactory; | ||
| import org.apache.iotdb.it.framework.IoTDBTestRunner; | ||
| import org.apache.iotdb.itbase.category.ClusterIT; | ||
| import org.apache.iotdb.itbase.category.LocalStandaloneIT; | ||
| import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; | ||
| import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; | ||
| import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; | ||
| import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; | ||
|
|
||
| import org.junit.AfterClass; | ||
| import org.junit.Assert; | ||
| import org.junit.BeforeClass; | ||
| import org.junit.Test; | ||
| import org.junit.experimental.categories.Category; | ||
| import org.junit.runner.RunWith; | ||
|
|
||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.sql.Connection; | ||
| import java.sql.ResultSet; | ||
| import java.sql.SQLException; | ||
| import java.sql.Statement; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| import static org.apache.iotdb.db.utils.constant.TestConstant.TIMESTAMP_STR; | ||
| import static org.junit.Assert.assertNotNull; | ||
|
|
||
| @RunWith(IoTDBTestRunner.class) | ||
| @Category({LocalStandaloneIT.class, ClusterIT.class}) | ||
| public class IoTDBRepairDataIT { | ||
|
|
||
| @BeforeClass | ||
| public static void setUp() throws Exception { | ||
| EnvFactory.getEnv().initClusterEnvironment(); | ||
| } | ||
|
|
||
| @AfterClass | ||
| public static void tearDown() throws Exception { | ||
| EnvFactory.getEnv().initClusterEnvironment(); | ||
| } | ||
|
|
||
| @Test | ||
| public void testRepairData() { | ||
| try (Connection connection = EnvFactory.getEnv().getConnection(); | ||
| Statement statement = connection.createStatement()) { | ||
| statement.execute("CREATE DATABASE root.tesgsg"); | ||
| statement.execute("CREATE TIMESERIES root.testsg.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN"); | ||
| File tsfile = generateUnsortedFile(); | ||
| statement.execute( | ||
| String.format("load \"%s\" verify=false", tsfile.getParentFile().getAbsolutePath())); | ||
|
|
||
| Assert.assertFalse(validate(statement)); | ||
| statement.execute("REPAIR DATA"); | ||
|
|
||
| int waitTimes = 20; | ||
| for (int i = 0; i < waitTimes; i++) { | ||
| boolean sorted = validate(statement); | ||
| if (sorted) { | ||
| return; | ||
| } | ||
| Thread.sleep(TimeUnit.SECONDS.toMillis(1)); | ||
| } | ||
| Assert.fail(); | ||
| } catch (Exception e) { | ||
| Assert.fail(e.getMessage()); | ||
| } | ||
| } | ||
|
|
||
| private File generateUnsortedFile() throws IOException { | ||
| Path tempDir = Files.createTempDirectory(""); | ||
| tempDir.toFile().deleteOnExit(); | ||
| String tsfileName = | ||
| TsFileNameGenerator.generateNewTsFileName(System.currentTimeMillis(), 1, 0, 0); | ||
| File tsfile = new File(tempDir + File.separator + tsfileName); | ||
| Files.createFile(tsfile.toPath()); | ||
|
|
||
| try (TsFileIOWriter writer = new TsFileIOWriter(tsfile)) { | ||
| writer.startChunkGroup("root.testsg.d1"); | ||
| ChunkWriterImpl chunkWriter = | ||
| new ChunkWriterImpl(new MeasurementSchema("s1", TSDataType.INT32)); | ||
| chunkWriter.write(2, 1); | ||
| chunkWriter.write(3, 1); | ||
| chunkWriter.write(5, 1); | ||
| chunkWriter.write(4, 1); | ||
| chunkWriter.sealCurrentPage(); | ||
|
|
||
| chunkWriter.writeToFileWriter(writer); | ||
| writer.endChunkGroup(); | ||
| writer.endFile(); | ||
| } | ||
| return tsfile; | ||
| } | ||
|
|
||
| private boolean validate(Statement statement) throws SQLException { | ||
| try (ResultSet resultSet = statement.executeQuery("SELECT s1 FROM root.testsg.d1")) { | ||
| assertNotNull(resultSet); | ||
| long time = Long.MIN_VALUE; | ||
| while (resultSet.next()) { | ||
| long currentTime = Long.parseLong(resultSet.getString(TIMESTAMP_STR)); | ||
| if (currentTime <= time) { | ||
| return false; | ||
| } | ||
| time = currentTime; | ||
| } | ||
| } | ||
| return true; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/UserDefinedEvent.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.iotdb.pipe.api.event; | ||
|
|
||
| /** | ||
| * User defined event is used to wrap data generated by users, keeping a source event to | ||
| * automatically report the processing progress to pipe engine. | ||
| */ | ||
| public abstract class UserDefinedEvent implements Event { | ||
|
|
||
| /** The user defined event is generated from this source event. */ | ||
| protected final Event sourceEvent; | ||
|
|
||
| /** | ||
| * @param sourceEvent The source event of this user defined event which is used to report the | ||
| * processing progress to pipe engine. Please notice that the source event should satisfy the | ||
| * following conditions: 1. A source event can only be assigned to one user defined event. 2. | ||
| * If more than one user defined events are generated from the same source event, only the | ||
| * last generated user defined event can be assigned with the source event, others should be | ||
| * assigned {@code null}, or call {@link #UserDefinedEvent()} to generate a user defined event | ||
| * without source event. | ||
| */ | ||
| protected UserDefinedEvent(Event sourceEvent) { | ||
| this.sourceEvent = parseRootSourceEvent(sourceEvent); | ||
| } | ||
|
|
||
| /** Generate a user defined event without source event. */ | ||
| protected UserDefinedEvent() { | ||
| this.sourceEvent = null; | ||
| } | ||
|
|
||
| private Event parseRootSourceEvent(Event sourceEvent) { | ||
| return sourceEvent instanceof UserDefinedEvent | ||
| ? ((UserDefinedEvent) sourceEvent).getSourceEvent() | ||
| : sourceEvent; | ||
| } | ||
|
|
||
| public Event getSourceEvent() { | ||
| return sourceEvent; | ||
| } | ||
| } |
Oops, something went wrong.