forked from apache/spark
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SC-5835][DIRECTORYCOMMIT] Move DirectoryCommitProtocol to its own pa…
…ckage ## What changes were proposed in this pull request? As part of the effort of clearly separating out Edge functionality, we're hereby moving all code related to the Directory Commit protocol to its own package. A few things about this change are not ideal - had to: - import com.databricks in `PartitioningAwareFileIndex` - open up `DirectoryAtomicReadProtocol.testingFs` (for testig hack) - write ugly code for getting configs from `SparkEnv.conf`, because of not having access to `ConfigEntry` - duplicate a bunch of utility classes: `Clock`, `ManualClock`, `SystemClock`, `ThreadUtils` ... but most of these (except the last) should hopefully be resolved by [SC-5838](https://databricks.atlassian.net/browse/SC-5838). ## How was this patch tested? spark-sql tests Author: Adrian Ionescu <adrian@databricks.com> Closes apache#247 from adrian-ionescu/SC-5835.
- Loading branch information
1 parent
0a21de2
commit 0985111
Showing
15 changed files
with
405 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
16 changes: 0 additions & 16 deletions
16
sql/core/src/main/scala/com/databricks/sql/transaction/ProtocolAlias.scala
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
114 changes: 114 additions & 0 deletions
114
sql/core/src/main/scala/com/databricks/util/Clock.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.databricks.util | ||
|
|
||
| /** | ||
| * An interface to represent clocks, so that they can be mocked out in unit tests. | ||
| */ | ||
| trait Clock { | ||
| def getTimeMillis(): Long | ||
| def waitTillTime(targetTime: Long): Long | ||
| } | ||
|
|
||
| /** | ||
| * A clock backed by the actual time from the OS as reported by the `System` API. | ||
| */ | ||
| class SystemClock extends Clock { | ||
|
|
||
| val minPollTime = 25L | ||
|
|
||
| /** | ||
| * @return the same time (milliseconds since the epoch) | ||
| * as is reported by `System.currentTimeMillis()` | ||
| */ | ||
| def getTimeMillis(): Long = System.currentTimeMillis() | ||
|
|
||
| /** | ||
| * @param targetTime block until the current time is at least this value | ||
| * @return current system time when wait has completed | ||
| */ | ||
| def waitTillTime(targetTime: Long): Long = { | ||
| var currentTime = 0L | ||
| currentTime = System.currentTimeMillis() | ||
|
|
||
| var waitTime = targetTime - currentTime | ||
| if (waitTime <= 0) { | ||
| return currentTime | ||
| } | ||
|
|
||
| val pollTime = math.max(waitTime / 10.0, minPollTime).toLong | ||
|
|
||
| while (true) { | ||
| currentTime = System.currentTimeMillis() | ||
| waitTime = targetTime - currentTime | ||
| if (waitTime <= 0) { | ||
| return currentTime | ||
| } | ||
| val sleepTime = math.min(waitTime, pollTime) | ||
| Thread.sleep(sleepTime) | ||
| } | ||
| -1 | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * A `Clock` whose time can be manually set and modified. Its reported time does not change | ||
| * as time elapses, but only as its time is modified by callers. This is mainly useful for | ||
| * testing. | ||
| * | ||
| * @param time initial time (in milliseconds since the epoch) | ||
| */ | ||
| class ManualClock(private var time: Long) extends Clock { | ||
|
|
||
| /** | ||
| * @return `ManualClock` with initial time 0 | ||
| */ | ||
| def this() = this(0L) | ||
|
|
||
| def getTimeMillis(): Long = | ||
| synchronized { | ||
| time | ||
| } | ||
|
|
||
| /** | ||
| * @param timeToSet new time (in milliseconds) that the clock should represent | ||
| */ | ||
| def setTime(timeToSet: Long): Unit = synchronized { | ||
| time = timeToSet | ||
| notifyAll() | ||
| } | ||
|
|
||
| /** | ||
| * @param timeToAdd time (in milliseconds) to add to the clock's time | ||
| */ | ||
| def advance(timeToAdd: Long): Unit = synchronized { | ||
| time += timeToAdd | ||
| notifyAll() | ||
| } | ||
|
|
||
| /** | ||
| * @param targetTime block until the clock time is set or advanced to at least this time | ||
| * @return current time reported by the clock when waiting finishes | ||
| */ | ||
| def waitTillTime(targetTime: Long): Long = synchronized { | ||
| while (time < targetTime) { | ||
| wait(10) | ||
| } | ||
| getTimeMillis() | ||
| } | ||
| } |
Oops, something went wrong.