-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Kernel]Add simple crc post commit for incremental crc writing. (#4134)
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md 2. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP] Your PR title ...'. 3. Be sure to keep the PR description updated to reflect all changes. 4. Please write your PR title to summarize what this PR proposes. 5. If possible, provide a concise example to reproduce the issue for a faster review. 6. If applicable, include the corresponding issue number in the PR title and link it in the body. --> #### Which Delta project/connector is this regarding? <!-- Please add the component selected below to the beginning of the pull request title For example: [Spark] Title of my pull request --> - [ ] Spark - [ ] Standalone - [ ] Flink - [X] Kernel - [ ] Other (fill in here) ## Description <!-- - Describe what this PR changes. - Describe why we need the change. If this PR resolves an issue be sure to include "Resolves #XXX" to correctly link and close the issue upon merge. --> This PR introduces a new post commit hook - ChecksumSimple, for writing CRC file after txn commit. CRC file will only be written only commit version - 1's snapshot reads CRC during state construction Other case will be handled in a separate PR ## How was this patch tested? <!-- If tests were added, say they were added here. Please make sure to test the changes thoroughly including negative and positive cases if possible. If the changes were tested in any way other than unit tests, please clarify how you tested step by step (ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future). If the changes were not tested, please explain why. --> E2e test ## Does this PR introduce _any_ user-facing changes? <!-- If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible. If possible, please also clarify if this is a user-facing change compared to the released Delta Lake versions or within the unreleased branches such as master. If no, write 'No'. --> No
- Loading branch information
1 parent
8045052
commit 7e85686
Showing
15 changed files
with
799 additions
and
91 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
164 changes: 164 additions & 0 deletions
164
kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,164 @@ | ||
| /* | ||
| * Copyright (2025) The Delta Lake Project Authors. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.delta.kernel.internal.checksum; | ||
|
|
||
| import static io.delta.kernel.internal.util.Preconditions.checkArgument; | ||
| import static java.util.Objects.requireNonNull; | ||
|
|
||
| import io.delta.kernel.data.ColumnVector; | ||
| import io.delta.kernel.data.ColumnarBatch; | ||
| import io.delta.kernel.data.Row; | ||
| import io.delta.kernel.internal.actions.Metadata; | ||
| import io.delta.kernel.internal.actions.Protocol; | ||
| import io.delta.kernel.internal.data.GenericRow; | ||
| import io.delta.kernel.internal.util.InternalUtils; | ||
| import io.delta.kernel.types.LongType; | ||
| import io.delta.kernel.types.StringType; | ||
| import io.delta.kernel.types.StructType; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| public class CRCInfo { | ||
| private static final Logger logger = LoggerFactory.getLogger(CRCInfo.class); | ||
|
|
||
| // Constants for schema field names | ||
| private static final String TABLE_SIZE_BYTES = "tableSizeBytes"; | ||
| private static final String NUM_FILES = "numFiles"; | ||
| private static final String NUM_METADATA = "numMetadata"; | ||
| private static final String NUM_PROTOCOL = "numProtocol"; | ||
| private static final String METADATA = "metadata"; | ||
| private static final String PROTOCOL = "protocol"; | ||
| private static final String TXN_ID = "txnId"; | ||
|
|
||
| public static final StructType CRC_FILE_SCHEMA = | ||
| new StructType() | ||
| .add(TABLE_SIZE_BYTES, LongType.LONG) | ||
| .add(NUM_FILES, LongType.LONG) | ||
| .add(NUM_METADATA, LongType.LONG) | ||
| .add(NUM_PROTOCOL, LongType.LONG) | ||
| .add(METADATA, Metadata.FULL_SCHEMA) | ||
| .add(PROTOCOL, Protocol.FULL_SCHEMA) | ||
| .add(TXN_ID, StringType.STRING, /*nullable*/ true); | ||
|
|
||
| public static Optional<CRCInfo> fromColumnarBatch( | ||
| long version, ColumnarBatch batch, int rowId, String crcFilePath) { | ||
| // Read required fields. | ||
| Protocol protocol = | ||
| Protocol.fromColumnVector(batch.getColumnVector(getSchemaIndex(PROTOCOL)), rowId); | ||
| Metadata metadata = | ||
| Metadata.fromColumnVector(batch.getColumnVector(getSchemaIndex(METADATA)), rowId); | ||
| long tableSizeBytes = | ||
| InternalUtils.requireNonNull( | ||
| batch.getColumnVector(getSchemaIndex(TABLE_SIZE_BYTES)), rowId, TABLE_SIZE_BYTES) | ||
| .getLong(rowId); | ||
| long numFiles = | ||
| InternalUtils.requireNonNull( | ||
| batch.getColumnVector(getSchemaIndex(NUM_FILES)), rowId, NUM_FILES) | ||
| .getLong(rowId); | ||
|
|
||
| // Read optional fields | ||
| ColumnVector txnIdColumnVector = batch.getColumnVector(getSchemaIndex(TXN_ID)); | ||
| Optional<String> txnId = | ||
| txnIdColumnVector.isNullAt(rowId) | ||
| ? Optional.empty() | ||
| : Optional.of(txnIdColumnVector.getString(rowId)); | ||
|
|
||
| // protocol and metadata are nullable per fromColumnVector's implementation. | ||
| if (protocol == null || metadata == null) { | ||
| logger.warn("Invalid checksum file missing protocol and/or metadata: {}", crcFilePath); | ||
| return Optional.empty(); | ||
| } | ||
| return Optional.of(new CRCInfo(version, metadata, protocol, tableSizeBytes, numFiles, txnId)); | ||
| } | ||
|
|
||
| private final long version; | ||
| private final Metadata metadata; | ||
| private final Protocol protocol; | ||
| private final long tableSizeBytes; | ||
| private final long numFiles; | ||
| private final Optional<String> txnId; | ||
|
|
||
| public CRCInfo( | ||
| long version, | ||
| Metadata metadata, | ||
| Protocol protocol, | ||
| long tableSizeBytes, | ||
| long numFiles, | ||
| Optional<String> txnId) { | ||
| checkArgument(tableSizeBytes >= 0); | ||
| checkArgument(numFiles >= 0); | ||
| this.version = version; | ||
| this.metadata = requireNonNull(metadata); | ||
| this.protocol = requireNonNull(protocol); | ||
| this.tableSizeBytes = tableSizeBytes; | ||
| this.numFiles = numFiles; | ||
| this.txnId = requireNonNull(txnId); | ||
| } | ||
|
|
||
| /** The version of the Delta table that this CRCInfo represents. */ | ||
| public long getVersion() { | ||
| return version; | ||
| } | ||
|
|
||
| /** The {@link Metadata} stored in this CRCInfo. */ | ||
| public Metadata getMetadata() { | ||
| return metadata; | ||
| } | ||
|
|
||
| /** The {@link Protocol} stored in this CRCInfo. */ | ||
| public Protocol getProtocol() { | ||
| return protocol; | ||
| } | ||
|
|
||
| public long getNumFiles() { | ||
| return numFiles; | ||
| } | ||
|
|
||
| public long getTableSizeBytes() { | ||
| return tableSizeBytes; | ||
| } | ||
|
|
||
| public Optional<String> getTxnId() { | ||
| return txnId; | ||
| } | ||
|
|
||
| /** | ||
| * Encode as a {@link Row} object with the schema {@link CRCInfo#CRC_FILE_SCHEMA}. | ||
| * | ||
| * @return {@link Row} object with the schema {@link CRCInfo#CRC_FILE_SCHEMA} | ||
| */ | ||
| public Row toRow() { | ||
| Map<Integer, Object> values = new HashMap<>(); | ||
| // Add required fields | ||
| values.put(getSchemaIndex(TABLE_SIZE_BYTES), tableSizeBytes); | ||
| values.put(getSchemaIndex(NUM_FILES), numFiles); | ||
| values.put(getSchemaIndex(NUM_METADATA), 1L); | ||
| values.put(getSchemaIndex(NUM_PROTOCOL), 1L); | ||
| values.put(getSchemaIndex(METADATA), metadata.toRow()); | ||
| values.put(getSchemaIndex(PROTOCOL), protocol.toRow()); | ||
|
|
||
| // Add optional fields | ||
| txnId.ifPresent(txn -> values.put(getSchemaIndex(TXN_ID), txn)); | ||
| return new GenericRow(CRC_FILE_SCHEMA, values); | ||
| } | ||
|
|
||
| private static int getSchemaIndex(String fieldName) { | ||
| return CRC_FILE_SCHEMA.indexOf(fieldName); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumWriter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| /* | ||
| * Copyright (2025) The Delta Lake Project Authors. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.delta.kernel.internal.checksum; | ||
|
|
||
| import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; | ||
| import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; | ||
| import static java.util.Objects.requireNonNull; | ||
|
|
||
| import io.delta.kernel.engine.Engine; | ||
| import io.delta.kernel.internal.fs.Path; | ||
| import io.delta.kernel.internal.util.FileNames; | ||
| import java.io.IOException; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| /** Writers for writing checksum files from a snapshot */ | ||
| public class ChecksumWriter { | ||
|
|
||
| private static final Logger logger = LoggerFactory.getLogger(ChecksumWriter.class); | ||
|
|
||
| private final Path logPath; | ||
|
|
||
| public ChecksumWriter(Path logPath) { | ||
| this.logPath = requireNonNull(logPath); | ||
| } | ||
|
|
||
| /** Writes a checksum file */ | ||
| public void writeCheckSum(Engine engine, CRCInfo crcInfo) throws IOException { | ||
| Path newChecksumPath = FileNames.checksumFile(logPath, crcInfo.getVersion()); | ||
| logger.info("Writing checksum file to path: {}", newChecksumPath); | ||
| wrapEngineExceptionThrowsIO( | ||
| () -> { | ||
| engine | ||
| .getJsonHandler() | ||
| .writeJsonFileAtomically( | ||
| newChecksumPath.toString(), | ||
| singletonCloseableIterator(crcInfo.toRow()), | ||
| false /* overwrite */); | ||
| logger.info("Write checksum file `{}` succeeds", newChecksumPath); | ||
| return null; | ||
| }, | ||
| "Write checksum file `%s`", | ||
| newChecksumPath); | ||
| } | ||
| } |
53 changes: 53 additions & 0 deletions
53
kernel/kernel-api/src/main/java/io/delta/kernel/internal/hook/ChecksumSimpleHook.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| /* | ||
| * Copyright (2025) The Delta Lake Project Authors. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.delta.kernel.internal.hook; | ||
|
|
||
| import static io.delta.kernel.internal.util.Preconditions.checkArgument; | ||
| import static java.util.Objects.requireNonNull; | ||
|
|
||
| import io.delta.kernel.engine.Engine; | ||
| import io.delta.kernel.hook.PostCommitHook; | ||
| import io.delta.kernel.internal.checksum.CRCInfo; | ||
| import io.delta.kernel.internal.checksum.ChecksumWriter; | ||
| import io.delta.kernel.internal.fs.Path; | ||
| import java.io.IOException; | ||
|
|
||
| /** | ||
| * A post-commit hook that writes a new checksum file at the version committed by the transaction. | ||
| * This hook performs a simple checksum operation without requiring previous checkpoint or log | ||
| * reading. | ||
| */ | ||
| public class ChecksumSimpleHook implements PostCommitHook { | ||
|
|
||
| private final CRCInfo crcInfo; | ||
| private final Path logPath; | ||
|
|
||
| public ChecksumSimpleHook(CRCInfo crcInfo, Path logPath) { | ||
| this.crcInfo = requireNonNull(crcInfo); | ||
| this.logPath = requireNonNull(logPath); | ||
| } | ||
|
|
||
| @Override | ||
| public void threadSafeInvoke(Engine engine) throws IOException { | ||
| checkArgument(engine != null); | ||
| new ChecksumWriter(logPath).writeCheckSum(engine, crcInfo); | ||
| } | ||
|
|
||
| @Override | ||
| public PostCommitHookType getType() { | ||
| return PostCommitHookType.CHECKSUM_SIMPLE; | ||
| } | ||
| } |
Oops, something went wrong.