perf: make index join/merge handlers optional

Restore the classes `JoinHandler` and `PropertiesMergeHandler` as the default handlers for the transformation functions Join, Groovy Join, Merge and Groovy Merge. The previous default handlers `IndexJoinHandler` and `IndexMergeHandler` have demonstrated negative impact on the performance of some transformations. Until this is better understood and fixed, these handlers are now optional and can be activated via the following Java properties and environment variables: - Join/Groovy Join: `hale.functions.use_index_join_handler` / `HALE_FUNCTIONS_USE_INDEX_JOIN_HANDLER` - Merge/Groovy Merge: `hale.functions.use_index_merge_handler` / `HALE_FUNCTIONS_USE_INDEX_MERGE_HANDLER` ING-4464
halestudio · Nov 11, 2024 · 1cb5163 · 1cb5163
1 parent 93c76ae
commit 1cb5163
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 8 deletions.
diff --git a/cst/plugins/eu.esdihumboldt.cst.functions.core/META-INF/MANIFEST.MF b/cst/plugins/eu.esdihumboldt.cst.functions.core/META-INF/MANIFEST.MF
@@ -9,6 +9,7 @@ Require-Bundle: eu.esdihumboldt.hale.common.align;bundle-version="2.2.0",
 Import-Package: com.google.common.base;version="9.0.0",
  com.google.common.collect,
  de.fhg.igd.osgi.util;version="1.0.0",
+ de.fhg.igd.slf4jplus,
  eu.esdihumboldt.hale.common.convert,
  eu.esdihumboldt.hale.common.core,
  eu.esdihumboldt.hale.common.core.io,
@@ -34,6 +35,7 @@ Import-Package: com.google.common.base;version="9.0.0",
  eu.esdihumboldt.util.groovy.paths,
  net.jcip.annotations,
  org.locationtech.jts.geom;version="1.13.0",
+ org.slf4j;version="1.7.36",
  org.springframework.core.convert;version="5.2.0"
 Export-Package: eu.esdihumboldt.cst.functions.core,
  eu.esdihumboldt.cst.functions.core.join,

diff --git a/...ugins/eu.esdihumboldt.cst.functions.core/src/eu/esdihumboldt/cst/functions/core/Join.java b/...ugins/eu.esdihumboldt.cst.functions.core/src/eu/esdihumboldt/cst/functions/core/Join.java
@@ -21,7 +21,10 @@
 import java.util.Collections;
 import java.util.List;
 
+import de.fhg.igd.slf4jplus.ALogger;
+import de.fhg.igd.slf4jplus.ALoggerFactory;
 import eu.esdihumboldt.cst.functions.core.join.IndexJoinHandler;
+import eu.esdihumboldt.cst.functions.core.join.JoinHandler;
 import eu.esdihumboldt.hale.common.align.model.Cell;
 import eu.esdihumboldt.hale.common.align.model.functions.JoinFunction;
 import eu.esdihumboldt.hale.common.align.model.functions.join.JoinParameter;
@@ -34,17 +37,38 @@
 /**
  * Type transformation that joins multiple instances of different source types
  * into one target instance, based on matching properties.
- * 
+ *
  * @author Kai Schwierczek
  */
 public class Join extends Retype implements JoinFunction, InstanceIndexContribution {
 
+	/**
+	 * The log
+	 */
+	private static final ALogger LOG = ALoggerFactory.getLogger(Join.class);
+
 	/**
 	 * @see eu.esdihumboldt.hale.common.align.transformation.function.impl.AbstractTypeTransformation#getInstanceHandler()
 	 */
 	@Override
 	public InstanceHandler<? super TransformationEngine> getInstanceHandler() {
-		return new IndexJoinHandler();
+		boolean useIndexJoinHandler = false;
+
+		String setting = System.getProperty("hale.functions.join.use_index_join_handler");
+
+		if (setting == null) {
+			setting = System.getenv("HALE_FUNCTIONS_USE_INDEX_JOIN_HANDLER");
+		}
+
+		if (setting != null) {
+			try {
+				useIndexJoinHandler = Boolean.valueOf(setting);
+			} catch (Throwable e) {
+				LOG.error("Error applying index join handler setting: " + setting, e);
+			}
+		}
+
+		return useIndexJoinHandler ? new IndexJoinHandler() : new JoinHandler();
 	}
 
 	/**

diff --git a/...gins/eu.esdihumboldt.cst.functions.core/src/eu/esdihumboldt/cst/functions/core/Merge.java b/...gins/eu.esdihumboldt.cst.functions.core/src/eu/esdihumboldt/cst/functions/core/Merge.java
@@ -20,7 +20,10 @@
 import java.util.Collections;
 import java.util.List;
 
+import de.fhg.igd.slf4jplus.ALogger;
+import de.fhg.igd.slf4jplus.ALoggerFactory;
 import eu.esdihumboldt.cst.functions.core.merge.IndexMergeHandler;
+import eu.esdihumboldt.cst.functions.core.merge.PropertiesMergeHandler;
 import eu.esdihumboldt.hale.common.align.model.Cell;
 import eu.esdihumboldt.hale.common.align.model.functions.MergeFunction;
 import eu.esdihumboldt.hale.common.align.model.functions.merge.MergeUtil;
@@ -32,17 +35,38 @@
 /**
  * Type transformation that merges multiple instances of the same source type
  * into one target instance, based on matching properties.
- * 
+ *
  * @author Simon Templer
  */
 public class Merge extends Retype implements MergeFunction, InstanceIndexContribution {
 
+	/**
+	 * The log
+	 */
+	private static final ALogger LOG = ALoggerFactory.getLogger(Merge.class);
+
 	/**
 	 * @see eu.esdihumboldt.hale.common.align.transformation.function.impl.AbstractTypeTransformation#getInstanceHandler()
 	 */
 	@Override
 	public InstanceHandler<? super TransformationEngine> getInstanceHandler() {
-		return new IndexMergeHandler();
+		boolean useIndexMergeHandler = false;
+
+		String setting = System.getProperty("hale.functions.use_index_merge_handler");
+
+		if (setting == null) {
+			setting = System.getenv("HALE_FUNCTIONS_USE_INDEX_MERGE_HANDLER");
+		}
+
+		if (setting != null) {
+			try {
+				useIndexMergeHandler = Boolean.valueOf(setting);
+			} catch (Throwable e) {
+				LOG.error("Error applying index merge handler setting: " + setting, e);
+			}
+		}
+
+		return useIndexMergeHandler ? new IndexMergeHandler() : new PropertiesMergeHandler();
 	}
 
 	/**

diff --git a/...sdihumboldt.cst.functions.groovy/src/eu/esdihumboldt/cst/functions/groovy/GroovyJoin.java b/...sdihumboldt.cst.functions.groovy/src/eu/esdihumboldt/cst/functions/groovy/GroovyJoin.java
@@ -18,8 +18,11 @@
 import java.util.Collection;
 import java.util.List;
 
+import de.fhg.igd.slf4jplus.ALogger;
+import de.fhg.igd.slf4jplus.ALoggerFactory;
 import eu.esdihumboldt.cst.functions.core.Join;
 import eu.esdihumboldt.cst.functions.core.join.IndexJoinHandler;
+import eu.esdihumboldt.cst.functions.core.join.JoinHandler;
 import eu.esdihumboldt.hale.common.align.model.Cell;
 import eu.esdihumboldt.hale.common.align.model.functions.JoinFunction;
 import eu.esdihumboldt.hale.common.align.model.impl.PropertyEntityDefinition;
@@ -32,11 +35,16 @@
  * into one target instance, based on matching properties. The transformation
  * also applies a Groovy script that can be used to control the target instance
  * creation.
- * 
+ *
  * @author Simon Templer
  */
 public class GroovyJoin extends GroovyRetype implements JoinFunction, InstanceIndexContribution {
 
+	/**
+	 * The log
+	 */
+	private static final ALogger LOG = ALoggerFactory.getLogger(GroovyJoin.class);
+
 	/**
 	 * The function ID. Not named <code>ID</code> to avoid shadowing
 	 * {@link JoinFunction#ID}.
@@ -50,7 +58,23 @@ public class GroovyJoin extends GroovyRetype implements JoinFunction, InstanceIn
 
 	@Override
 	public InstanceHandler<? super TransformationEngine> getInstanceHandler() {
-		return new IndexJoinHandler();
+		boolean useIndexJoinHandler = false;
+
+		String setting = System.getProperty("hale.functions.use_index_join_handler");
+
+		if (setting == null) {
+			setting = System.getenv("HALE_FUNCTIONS_USE_INDEX_JOIN_HANDLER");
+		}
+
+		if (setting != null) {
+			try {
+				useIndexJoinHandler = Boolean.valueOf(setting);
+			} catch (Throwable e) {
+				LOG.error("Error applying index join handler setting: " + setting, e);
+			}
+		}
+
+		return useIndexJoinHandler ? new IndexJoinHandler() : new JoinHandler();
 	}
 
 	/**

diff --git a/...dihumboldt.cst.functions.groovy/src/eu/esdihumboldt/cst/functions/groovy/GroovyMerge.java b/...dihumboldt.cst.functions.groovy/src/eu/esdihumboldt/cst/functions/groovy/GroovyMerge.java
@@ -19,8 +19,11 @@
 import java.util.Collection;
 import java.util.List;
 
+import de.fhg.igd.slf4jplus.ALogger;
+import de.fhg.igd.slf4jplus.ALoggerFactory;
 import eu.esdihumboldt.cst.functions.core.Merge;
 import eu.esdihumboldt.cst.functions.core.merge.IndexMergeHandler;
+import eu.esdihumboldt.cst.functions.core.merge.PropertiesMergeHandler;
 import eu.esdihumboldt.hale.common.align.model.Cell;
 import eu.esdihumboldt.hale.common.align.model.functions.MergeFunction;
 import eu.esdihumboldt.hale.common.align.model.impl.PropertyEntityDefinition;
@@ -33,11 +36,16 @@
  * into one target instance, based on matching properties. The transformation
  * also applies a Groovy script that can be used to control the target instance
  * creation.
- * 
+ *
  * @author Simon Templer
  */
 public class GroovyMerge extends GroovyRetype implements MergeFunction, InstanceIndexContribution {
 
+	/**
+	 * The log
+	 */
+	private static final ALogger LOG = ALoggerFactory.getLogger(GroovyMerge.class);
+
 	/**
 	 * The function ID. Not named <code>ID</code> to avoid shadowing
 	 * {@link MergeFunction#ID}.
@@ -51,7 +59,23 @@ public class GroovyMerge extends GroovyRetype implements MergeFunction, Instance
 
 	@Override
 	public InstanceHandler<? super TransformationEngine> getInstanceHandler() {
-		return new IndexMergeHandler();
+		boolean useIndexMergeHandler = false;
+
+		String setting = System.getProperty("hale.functions.use_index_merge_handler");
+
+		if (setting == null) {
+			setting = System.getenv("HALE_FUNCTIONS_USE_INDEX_MERGE_HANDLER");
+		}
+
+		if (setting != null) {
+			try {
+				useIndexMergeHandler = Boolean.valueOf(setting);
+			} catch (Throwable e) {
+				LOG.error("Error applying index merge handler setting: " + setting, e);
+			}
+		}
+
+		return useIndexMergeHandler ? new IndexMergeHandler() : new PropertiesMergeHandler();
 	}
 
 	/**