From 79cbf317ea6dbd280d3b6cb537d3ec85f960aa0e Mon Sep 17 00:00:00 2001 From: Song Qi <37234782+JustDoCoder@users.noreply.github.com> Date: Fri, 29 Jan 2021 10:31:19 +0800 Subject: [PATCH 1/3] Create BlockManager.scala --- src/main/scala/org/apache/spark/storage/BlockManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/apache/spark/storage/BlockManager.scala b/src/main/scala/org/apache/spark/storage/BlockManager.scala index 31b69800..344c8686 100644 --- a/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -190,7 +190,7 @@ private[spark] class BlockManager( val numNum = conf.getInt("spark.yarn.numa.num", 2) if (pmemMode.equals("AppDirect")) { - if (!isDriver && pmemInitialPaths.size > 1) { + if (!isDriver && pmemInitialPaths.size >= 1) { if (numaNodeId == -1) { numaNodeId = executorId.toInt } From 2c213ff223d6394eca9e1283a7a0b6c6fd8ad7f1 Mon Sep 17 00:00:00 2001 From: Song Qi <37234782+JustDoCoder@users.noreply.github.com> Date: Tue, 2 Feb 2021 17:11:04 +0800 Subject: [PATCH 2/3] Update BlockManager.scala --- src/main/scala/org/apache/spark/storage/BlockManager.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/apache/spark/storage/BlockManager.scala b/src/main/scala/org/apache/spark/storage/BlockManager.scala index 93c0780c..7c5118c5 100644 --- a/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -210,8 +210,8 @@ private[spark] class BlockManager( } require(file.isDirectory(), "PMem directory is required for initialization") - PersistentMemoryPlatform.initialize(initPath, pmemInitialSize, 0) - logInfo(s"Intel Optane PMem initialized with path: ${initPath}, size: ${pmemInitialSize} ") + PersistentMemoryPlatform.initialize(path_postfix, pmemInitialSize, 0) + logInfo(s"Intel Optane PMem initialized with path: ${path_postfix}, size: ${pmemInitialSize} ") } } else if (pmemMode.equals("KMemDax")) { if (!isDriver) { From 21fae645859e3daf60f35f7f5cf34701760e25d9 Mon Sep 17 00:00:00 2001 From: Song Qi <37234782+JustDoCoder@users.noreply.github.com> Date: Wed, 3 Feb 2021 09:11:17 +0800 Subject: [PATCH 3/3] Update BlockManager.scala --- src/main/scala/org/apache/spark/storage/BlockManager.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/apache/spark/storage/BlockManager.scala b/src/main/scala/org/apache/spark/storage/BlockManager.scala index 7c5118c5..81e7047d 100644 --- a/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -210,8 +210,8 @@ private[spark] class BlockManager( } require(file.isDirectory(), "PMem directory is required for initialization") - PersistentMemoryPlatform.initialize(path_postfix, pmemInitialSize, 0) - logInfo(s"Intel Optane PMem initialized with path: ${path_postfix}, size: ${pmemInitialSize} ") + PersistentMemoryPlatform.initialize(file.getAbsolutePath, pmemInitialSize, 0) + logInfo(s"Intel Optane PMem initialized with path: ${file.getAbsolutePath}, size: ${pmemInitialSize} ") } } else if (pmemMode.equals("KMemDax")) { if (!isDriver) {