Skip to content

Commit

Permalink
Replace with @tempdir
Browse files Browse the repository at this point in the history
  • Loading branch information
@kaijchen
kaijchen committed Jan 17, 2023
1 parent da00624 commit 26178ab
Show file tree
Hide file tree
Showing 11 changed files with 23 additions and 45 deletions.
6 changes: 2 additions & 4 deletions ...mr/src/test/java/org/apache/hadoop/mapreduce/task/reduce/RssInMemoryRemoteMergerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.util.TreeMap;

import com.google.common.collect.Lists;
import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
Expand All @@ -47,19 +46,18 @@
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.util.Progress;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

public class RssInMemoryRemoteMergerTest {

@Test
public void mergerTest() throws IOException {
public void mergerTest(@TempDir File tmpDir) throws IOException {
JobConf jobConf = new JobConf();
FileSystem fs = FileSystem.getLocal(jobConf);
LocalDirAllocator lda = new LocalDirAllocator(MRConfig.LOCAL_DIR);
File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
JobID jobId = new JobID("a", 0);
final TaskAttemptID mapId1 = new TaskAttemptID(
new TaskID(jobId, TaskType.MAP, 1), 0);
Expand Down
6 changes: 2 additions & 4 deletions ...t-mr/src/test/java/org/apache/hadoop/mapreduce/task/reduce/RssRemoteMergeManagerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import java.util.TreeMap;

import com.google.common.collect.Lists;
import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
Expand All @@ -47,6 +46,7 @@
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.util.Progress;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import static org.junit.jupiter.api.Assertions.assertEquals;

Expand All @@ -62,13 +62,11 @@ public class RssRemoteMergeManagerTest {
new TaskID(jobId, TaskType.REDUCE, 0), 0);

@Test
public void mergerTest() throws Throwable {
public void mergerTest(@TempDir File tmpDir) throws Throwable {
JobConf jobConf = new JobConf();
final FileSystem fs = FileSystem.getLocal(jobConf);
final LocalDirAllocator lda = new LocalDirAllocator(MRConfig.LOCAL_DIR);

File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
jobConf.set("mapreduce.reduce.memory.totalbytes", "1024");
jobConf.set("mapreduce.reduce.shuffle.memory.limit.percent", "0.01");
jobConf.set("mapreduce.reduce.shuffle.merge.percent", "0.1");
Expand Down
6 changes: 2 additions & 4 deletions ...-common/src/test/java/org/apache/uniffle/test/RepartitionWithHdfsMultiStorageRssTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,18 @@
import java.util.Map;

import com.google.common.collect.Maps;
import com.google.common.io.Files;
import org.apache.spark.SparkConf;
import org.apache.spark.shuffle.RssSparkConfig;
import org.junit.jupiter.api.BeforeAll;

import org.apache.uniffle.coordinator.CoordinatorConf;
import org.apache.uniffle.server.ShuffleServerConf;
import org.apache.uniffle.storage.util.StorageType;
import org.junit.jupiter.api.io.TempDir;

public class RepartitionWithHdfsMultiStorageRssTest extends RepartitionTest {
@BeforeAll
public static void setupServers() throws Exception {
public static void setupServers(@TempDir File tmpDir) throws Exception {
CoordinatorConf coordinatorConf = getCoordinatorConf();
Map<String, String> dynamicConf = Maps.newHashMap();
dynamicConf.put(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_PATH.key(), HDFS_URI + "rss/test");
Expand All @@ -43,8 +43,6 @@ public static void setupServers() throws Exception {

ShuffleServerConf shuffleServerConf = getShuffleServerConf();
// local storage config
File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
File dataDir1 = new File(tmpDir, "data1");
File dataDir2 = new File(tmpDir, "data2");
String basePath = dataDir1.getAbsolutePath() + "," + dataDir2.getAbsolutePath();
Expand Down
5 changes: 2 additions & 3 deletions ...t/spark-common/src/test/java/org/apache/uniffle/test/RepartitionWithLocalFileRssTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.util.concurrent.TimeUnit;

import com.google.common.collect.Maps;
import com.google.common.io.Files;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.spark.SparkConf;
import org.apache.spark.shuffle.RssSparkConfig;
Expand All @@ -34,20 +33,20 @@
import org.apache.uniffle.coordinator.CoordinatorConf;
import org.apache.uniffle.server.ShuffleServerConf;
import org.apache.uniffle.storage.util.StorageType;
import org.junit.jupiter.api.io.TempDir;

import static org.apache.uniffle.common.config.RssClientConf.COMPRESSION_TYPE;

public class RepartitionWithLocalFileRssTest extends RepartitionTest {

@BeforeAll
public static void setupServers() throws Exception {
public static void setupServers(@TempDir File tmpDir) throws Exception {
CoordinatorConf coordinatorConf = getCoordinatorConf();
Map<String, String> dynamicConf = Maps.newHashMap();
dynamicConf.put(RssSparkConfig.RSS_STORAGE_TYPE.key(), StorageType.LOCALFILE.name());
addDynamicConf(coordinatorConf, dynamicConf);
createCoordinatorServer(coordinatorConf);
ShuffleServerConf shuffleServerConf = getShuffleServerConf();
File tmpDir = Files.createTempDir();
File dataDir1 = new File(tmpDir, "data1");
File dataDir2 = new File(tmpDir, "data2");
String basePath = dataDir1.getAbsolutePath() + "," + dataDir2.getAbsolutePath();
Expand Down
6 changes: 2 additions & 4 deletions ...ommon/src/test/java/org/apache/uniffle/test/RepartitionWithMemoryMultiStorageRssTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,18 @@
import java.util.Map;

import com.google.common.collect.Maps;
import com.google.common.io.Files;
import org.apache.spark.SparkConf;
import org.apache.spark.shuffle.RssSparkConfig;
import org.junit.jupiter.api.BeforeAll;

import org.apache.uniffle.coordinator.CoordinatorConf;
import org.apache.uniffle.server.ShuffleServerConf;
import org.apache.uniffle.storage.util.StorageType;
import org.junit.jupiter.api.io.TempDir;

public class RepartitionWithMemoryMultiStorageRssTest extends RepartitionTest {
@BeforeAll
public static void setupServers() throws Exception {
public static void setupServers(@TempDir File tmpDir) throws Exception {
CoordinatorConf coordinatorConf = getCoordinatorConf();
Map<String, String> dynamicConf = Maps.newHashMap();
dynamicConf.put(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_PATH.key(), HDFS_URI + "rss/test");
Expand All @@ -43,8 +43,6 @@ public static void setupServers() throws Exception {
ShuffleServerConf shuffleServerConf = getShuffleServerConf();

// local storage config
File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
File dataDir1 = new File(tmpDir, "data1");
File dataDir2 = new File(tmpDir, "data2");
String basePath = dataDir1.getAbsolutePath() + "," + dataDir2.getAbsolutePath();
Expand Down
6 changes: 2 additions & 4 deletions ...test/spark-common/src/test/java/org/apache/uniffle/test/RepartitionWithMemoryRssTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import java.util.Map;

import com.google.common.collect.Maps;
import com.google.common.io.Files;
import org.apache.spark.SparkConf;
import org.apache.spark.shuffle.RssSparkConfig;
import org.junit.jupiter.api.BeforeAll;
Expand All @@ -31,11 +30,12 @@
import org.apache.uniffle.coordinator.CoordinatorConf;
import org.apache.uniffle.server.ShuffleServerConf;
import org.apache.uniffle.storage.util.StorageType;
import org.junit.jupiter.api.io.TempDir;

public class RepartitionWithMemoryRssTest extends RepartitionTest {

@BeforeAll
public static void setupServers() throws Exception {
public static void setupServers(@TempDir File tmpDir) throws Exception {
CoordinatorConf coordinatorConf = getCoordinatorConf();
coordinatorConf.set(CoordinatorConf.COORDINATOR_APP_EXPIRED, 5000L);
Map<String, String> dynamicConf = Maps.newHashMap();
Expand All @@ -45,8 +45,6 @@ public static void setupServers() throws Exception {
ShuffleServerConf shuffleServerConf = getShuffleServerConf();
shuffleServerConf.set(ShuffleServerConf.SERVER_HEARTBEAT_INTERVAL, 5000L);
shuffleServerConf.set(ShuffleServerConf.SERVER_APP_EXPIRED_WITHOUT_HEARTBEAT, 4000L);
File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
File dataDir1 = new File(tmpDir, "data1");
File dataDir2 = new File(tmpDir, "data2");
String basePath = dataDir1.getAbsolutePath() + "," + dataDir2.getAbsolutePath();
Expand Down
5 changes: 2 additions & 3 deletions ...ion-test/spark-common/src/test/java/org/apache/uniffle/test/SparkClientWithLocalTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.roaringbitmap.longlong.LongIterator;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

Expand Down Expand Up @@ -62,11 +62,10 @@ public class SparkClientWithLocalTest extends ShuffleReadWriteBase {
Lists.newArrayList(new ShuffleServerInfo(LOCALHOST, SHUFFLE_SERVER_PORT));

@BeforeAll
public static void setupServers() throws Exception {
public static void setupServers(@TempDir File tmpDir) throws Exception {
CoordinatorConf coordinatorConf = getCoordinatorConf();
createCoordinatorServer(coordinatorConf);
ShuffleServerConf shuffleServerConf = getShuffleServerConf();
File tmpDir = Files.createTempDir();
DATA_DIR1 = new File(tmpDir, "data1");
DATA_DIR2 = new File(tmpDir, "data2");
String basePath = DATA_DIR1.getAbsolutePath() + "," + DATA_DIR2.getAbsolutePath();
Expand Down
6 changes: 2 additions & 4 deletions ...rk-common/src/test/java/org/apache/uniffle/test/SparkSQLWithDelegationShuffleManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.util.concurrent.TimeUnit;

import com.google.common.collect.Maps;
import com.google.common.io.Files;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.spark.SparkConf;
import org.apache.spark.shuffle.RssSparkConfig;
Expand All @@ -33,11 +32,12 @@
import org.apache.uniffle.coordinator.CoordinatorConf;
import org.apache.uniffle.server.ShuffleServerConf;
import org.apache.uniffle.storage.util.StorageType;
import org.junit.jupiter.api.io.TempDir;

public class SparkSQLWithDelegationShuffleManager extends SparkSQLTest {

@BeforeAll
public static void setupServers() throws Exception {
public static void setupServers(@TempDir File tmpDir) throws Exception {
final String candidates = Objects.requireNonNull(
SparkSQLWithDelegationShuffleManager.class.getClassLoader().getResource("candidates")).getFile();
CoordinatorConf coordinatorConf = getCoordinatorConf();
Expand All @@ -55,8 +55,6 @@ public static void setupServers() throws Exception {
ShuffleServerConf shuffleServerConf = getShuffleServerConf();
shuffleServerConf.set(ShuffleServerConf.SERVER_HEARTBEAT_INTERVAL, 1000L);
shuffleServerConf.set(ShuffleServerConf.SERVER_APP_EXPIRED_WITHOUT_HEARTBEAT, 4000L);
File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
File dataDir1 = new File(tmpDir, "data1");
File dataDir2 = new File(tmpDir, "data2");
String basePath = dataDir1.getAbsolutePath() + "," + dataDir2.getAbsolutePath();
Expand Down
6 changes: 2 additions & 4 deletions ...n/src/test/java/org/apache/uniffle/test/SparkSQLWithDelegationShuffleManagerFallback.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.util.concurrent.TimeUnit;

import com.google.common.collect.Maps;
import com.google.common.io.Files;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.spark.SparkConf;
import org.apache.spark.shuffle.RssSparkConfig;
Expand All @@ -33,11 +32,12 @@
import org.apache.uniffle.coordinator.CoordinatorConf;
import org.apache.uniffle.server.ShuffleServerConf;
import org.apache.uniffle.storage.util.StorageType;
import org.junit.jupiter.api.io.TempDir;

public class SparkSQLWithDelegationShuffleManagerFallback extends SparkSQLTest {

@BeforeAll
public static void setupServers() throws Exception {
public static void setupServers(@TempDir File tmpDir) throws Exception {
final String candidates = Objects.requireNonNull(
SparkSQLWithDelegationShuffleManager.class.getClassLoader().getResource("candidates")).getFile();
CoordinatorConf coordinatorConf = getCoordinatorConf();
Expand All @@ -55,8 +55,6 @@ public static void setupServers() throws Exception {
ShuffleServerConf shuffleServerConf = getShuffleServerConf();
shuffleServerConf.set(ShuffleServerConf.SERVER_HEARTBEAT_INTERVAL, 1000L);
shuffleServerConf.set(ShuffleServerConf.SERVER_APP_EXPIRED_WITHOUT_HEARTBEAT, 4000L);
File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
File dataDir1 = new File(tmpDir, "data1");
File dataDir2 = new File(tmpDir, "data2");
String basePath = dataDir1.getAbsolutePath() + "," + dataDir2.getAbsolutePath();
Expand Down
6 changes: 2 additions & 4 deletions ...-test/spark-common/src/test/java/org/apache/uniffle/test/SparkSQLWithMemoryLocalTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@
import java.util.Map;

import com.google.common.collect.Maps;
import com.google.common.io.Files;
import org.apache.spark.SparkConf;
import org.apache.spark.shuffle.RssSparkConfig;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.io.TempDir;

import org.apache.uniffle.coordinator.CoordinatorConf;
import org.apache.uniffle.server.ShuffleServerConf;
Expand All @@ -37,7 +37,7 @@ public class SparkSQLWithMemoryLocalTest extends SparkSQLTest {
private static String basePath;

@BeforeAll
public static void setupServers() throws Exception {
public static void setupServers(@TempDir File tmpDir) throws Exception {
CoordinatorConf coordinatorConf = getCoordinatorConf();
coordinatorConf.setLong("rss.coordinator.app.expired", 5000);
Map<String, String> dynamicConf = Maps.newHashMap();
Expand All @@ -47,8 +47,6 @@ public static void setupServers() throws Exception {
ShuffleServerConf shuffleServerConf = getShuffleServerConf();
shuffleServerConf.setLong("rss.server.heartbeat.interval", 5000);
shuffleServerConf.setLong("rss.server.app.expired.withoutHeartbeat", 4000);
File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
File dataDir1 = new File(tmpDir, "data1");
File dataDir2 = new File(tmpDir, "data2");
basePath = dataDir1.getAbsolutePath() + "," + dataDir2.getAbsolutePath();
Expand Down
10 changes: 3 additions & 7 deletions storage/src/test/java/org/apache/uniffle/storage/handler/impl/LocalFileHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@

import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.junit.jupiter.api.Test;

import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.common.ShuffleIndexResult;
import org.apache.uniffle.common.config.RssBaseConf;
import org.apache.uniffle.storage.util.ShuffleStorageUtils;
import org.junit.jupiter.api.io.TempDir;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
Expand All @@ -40,9 +40,7 @@
public class LocalFileHandlerTest {

@Test
public void writeTest() throws Exception {
File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
public void writeTest(@TempDir File tmpDir) throws Exception {
File dataDir1 = new File(tmpDir, "data1");
File dataDir2 = new File(tmpDir, "data2");
String[] basePaths = new String[]{dataDir1.getAbsolutePath(),
Expand Down Expand Up @@ -103,9 +101,7 @@ public void writeTest() throws Exception {
}

@Test
public void writeBigDataTest() throws IOException {
File tmpDir = Files.createTempDir();
tmpDir.deleteOnExit();
public void writeBigDataTest(@TempDir File tmpDir) throws IOException {
File writeFile = new File(tmpDir, "writetest");
LocalFileWriter writer = new LocalFileWriter(writeFile);
int size = Integer.MAX_VALUE / 100;
Expand Down

0 comments on commit 26178ab

Please sign in to comment.