[Follow-up] Fix log4j and some dependencies for shims-common (#362) (…

…#363) * log4j and some fix in common * remove redirectLog() * nit Co-authored-by: Yizhong Zhang <zyzzxycj@163.com>
oap-project · Jul 10, 2023 · e7c2c74 · e7c2c74
1 parent a73ea2a
commit e7c2c74
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 25 deletions.
diff --git a/core/pom.xml b/core/pom.xml
@@ -63,6 +63,10 @@
             <groupId>org.apache.ivy</groupId>
             <artifactId>ivy</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>

diff --git a/core/raydp-main/pom.xml b/core/raydp-main/pom.xml
@@ -61,6 +61,10 @@
           <groupId>org.apache.ivy</groupId>
           <artifactId>ivy</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>

diff --git a/core/raydp-main/src/main/scala/org/apache/spark/executor/RayDPExecutor.scala b/core/raydp-main/src/main/scala/org/apache/spark/executor/RayDPExecutor.scala
@@ -30,7 +30,6 @@ import io.ray.runtime.config.RayConfig
 import org.apache.arrow.vector.ipc.{ArrowStreamWriter, WriteChannel}
 import org.apache.arrow.vector.ipc.message.{IpcOption, MessageSerializer}
 import org.apache.arrow.vector.types.pojo.Schema
-import org.apache.log4j.{FileAppender => Log4jFileAppender, _}
 
 import org.apache.spark._
 import org.apache.spark.deploy.SparkHadoopUtil
@@ -111,7 +110,6 @@ class RayDPExecutor(
     }
     createWorkingDir(appId)
     setUserDir()
-    // redirectLog()
 
     val userClassPath = classPathEntries.split(java.io.File.pathSeparator)
       .filter(_.nonEmpty).map(new File(_).toURI.toURL)
@@ -251,27 +249,6 @@ class RayDPExecutor(
     }
   }
 
-  def redirectLog(): Unit = {
-    val logFile = Paths.get(workingDir.getAbsolutePath, s"executor${executorId}.out")
-    val errorFile = Paths.get(workingDir.getAbsolutePath, s"executor${executorId}.err")
-    logInfo(s"Redirect executor log to ${logFile.toString}")
-    val appenders = LogManager.getRootLogger.getAllAppenders
-    // There should be a console appender. Use its layout.
-    val defaultAppender = appenders.nextElement().asInstanceOf[Appender]
-    val layout = defaultAppender.getLayout
-
-    val out = new Log4jFileAppender(layout, logFile.toString)
-    out.setName("outfile")
-
-    val err = new Log4jFileAppender(layout, errorFile.toString())
-    err.setName("errfile")
-    err.setThreshold(Level.ERROR)
-
-    LogManager.getRootLogger.addAppender(out)
-    LogManager.getRootLogger.addAppender(err)
-    LogManager.getRootLogger.removeAppender(defaultAppender)
-  }
-
   def createTemporaryRpcEnv(
       name: String,
       conf: SparkConf): Unit = {

diff --git a/core/shims/common/pom.xml b/core/shims/common/pom.xml
@@ -66,11 +66,32 @@
             <scope>provided</scope>
             <exclusions>
                 <exclusion>
-                <groupId>org.apache.commons</groupId>
-                <artifactId>commons-text</artifactId>
+                    <groupId>org.xerial.snappy</groupId>
+                    <artifactId>snappy-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-compress</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-text</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.ivy</groupId>
+                    <artifactId>ivy</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <version>${snappy.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-text</artifactId>
@@ -81,5 +102,15 @@
             <artifactId>protobuf-java</artifactId>
             <version>${protobuf.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.ivy</groupId>
+            <artifactId>ivy</artifactId>
+            <version>${ivy.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>${commons.compress.version}</version>
+        </dependency>
     </dependencies>
 </project>
diff --git a/core/shims/spark322/pom.xml b/core/shims/spark322/pom.xml
@@ -97,6 +97,10 @@
           <groupId>org.apache.ivy</groupId>
           <artifactId>ivy</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>