remove controlNonVisibleCharacters - this is a frontend job

terrier-org · Jan 10, 2025 · e5802e5 · e5802e5
1 parent dd3cb71
commit e5802e5
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 8 deletions.
diff --git a/modules/core/src/main/java/org/terrier/querying/SimpleDecorate.java b/modules/core/src/main/java/org/terrier/querying/SimpleDecorate.java
@@ -37,13 +37,11 @@
 public class SimpleDecorate implements PostFilter {
 
 	static Logger logger = LoggerFactory.getLogger(SimpleDecorate.class);
-	protected static final Pattern controlNonVisibleCharacters = Pattern.compile("[\\p{Cntrl}\uFFFD\uFFFF]|[^\\p{Graph}\\p{Space}]");
 
 	MetaIndex meta = null;
 	String[] decorateKeys = null;
 	boolean any = false;
 
-	Matcher controlNonVisibleCharactersMatcher = controlNonVisibleCharacters.matcher("");
 	/** 
 	 * Adds all the metadata for the specified document occurring at the specified
 	 * rank to the ResultSet
@@ -61,8 +59,7 @@ public final byte filter(
 			final String[] values = meta.getItems(decorateKeys, docid);
 			for(int j=0;j<decorateKeys.length;j++)
 			{
-				controlNonVisibleCharactersMatcher.reset(values[j]);
-				results.addMetaItem(decorateKeys[j], rank, controlNonVisibleCharactersMatcher.replaceAll(""));
+				results.addMetaItem(decorateKeys[j], rank, values[i]);
 			}
 			return PostFilter.FILTER_OK;
 		} catch (Exception e) {

diff --git a/modules/core/src/main/java/org/terrier/querying/SimpleDecorateProcess.java b/modules/core/src/main/java/org/terrier/querying/SimpleDecorateProcess.java
@@ -57,7 +57,7 @@ public void process(Manager manager, Request q) {
 				: ArrayUtils.parseCommaDelimitedString(q.getControl("decorate"));
 			logger.debug("Decorating for " + java.util.Arrays.toString(decorateKeys));
 			if (decorateKeys.length == 0){
-				logger.warn("SimpleDecorate called, but no meta keys detected - either metaindex is empty, or decorate control is empty");
+				logger.warn("SimpleDecorate called, but no meta keys detected - either metaindex is empty, or decorate control is empty");
 				return;
 			}
 			ResultSet res = q.getResultSet();
@@ -69,9 +69,7 @@ public void process(Manager manager, Request q) {
 				String[] finalmeta = new String[num_docs];
 				for (int i=0;i<num_docs;i++)
 				{
-					String value = meta[i][j];
-					controlNonVisibleCharactersMatcher.reset(value);
-					finalmeta[i] = controlNonVisibleCharactersMatcher.replaceAll("");
+					finalmeta[i] = meta[i][j];
 				}
 				res.addMetaItems(decorateKeys[j], finalmeta);
 			}