Now using automated and full categories path

open4good · Oct 3, 2024 · ff48af0 · ff48af0
1 parent 3dd6ff6
commit ff48af0
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 30 deletions.
diff --git a/...s/src/main/java/org/open4goods/commons/config/yml/datasource/CsvDataSourceProperties.java b/...s/src/main/java/org/open4goods/commons/config/yml/datasource/CsvDataSourceProperties.java
@@ -105,10 +105,6 @@ public class CsvDataSourceProperties {
 	 */
 	private Boolean trimUrlParameters = false;
 
-	/**
-	 * Column name indicating the productCategory
-	 */
-	protected List<String> productTags = new ArrayList<>();
 
 	/**
 	 * If true, all columns of this datasource will ba added as attributes
@@ -287,15 +283,6 @@ public void setTrimUrlParameters(final Boolean trimUrlParameters) {
 		this.trimUrlParameters = trimUrlParameters;
 	}
 
-	public List<String> getProductTags() {
-		return productTags;
-	}
-
-	public void setProductTags(List<String> productTags) {
-		this.productTags = productTags;
-	}
-
-
 
 	public Set<String> getPrice() {
 		return price;

diff --git a/commons/src/main/java/org/open4goods/commons/model/data/DataFragment.java b/commons/src/main/java/org/open4goods/commons/model/data/DataFragment.java
@@ -656,7 +656,7 @@ public void addRseRating(final Rating rr) throws ValidationException {
 
 	public void addProductTags(final List<String> tags) {
 
-		addProductTag(StringUtils.join(tags, ">").toUpperCase().trim());
+		addProductTag(StringUtils.join(tags, " | ").toUpperCase().trim());
 	}
 
 	public void addProductTag(final String category) {

diff --git a/crawler/src/main/java/org/open4goods/crawler/services/fetching/CsvIndexationWorker.java b/crawler/src/main/java/org/open4goods/crawler/services/fetching/CsvIndexationWorker.java
@@ -6,6 +6,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -418,7 +419,7 @@ private DataFragment parseCsvLine(final DataFragmentWebCrawler crawler, final Cr
 		}
 
 		p.addName(getFromCsvRow(item, csvProperties.getName()));
-		p.addProductTags(getCategoryFromCsvRows(item, csvProperties.getProductTags()));
+		p.addProductTags(getCategoryFromCsvRows(item));
 
 		if (!StringUtils.isEmpty(csvProperties.getAttrs())) {
 			handleAttributes(p, config, getFromCsvRow(item, csvProperties.getAttrs()), dedicatedLogger);
@@ -677,24 +678,21 @@ private void handleAttributes(final DataFragment pd, final DataSourceProperties
 	 */
 	private String getFromCsvRow(final Map<String, String> item, final String colName) {
 
-		final String val = item.get(colName);
-		if (null != val) {
-			return val;
-
-		} else {
-			return null;
-		}
+		return item.get(colName);
+
 	}
 
-	private List<String> getCategoryFromCsvRows(Map<String, String> item, List<String> colNames) {
-		List<String> ret = new ArrayList<>();
+	private List<String> getCategoryFromCsvRows(Map<String, String> item) {
+		Set<String> catsColumns = new HashSet<String>(item.keySet());
+
+		List<String> ret = catsColumns.stream()
+							.filter(e -> e.toLowerCase().contains("categor"))
+							.sorted()
+							.map(e -> item.get(e) )
+							.filter(e -> !StringUtils.isEmpty(e))
+							.toList();
+
 
-		for (String colName : colNames) {
-			String val = getFromCsvRow(item, colName);
-			if (!StringUtils.isEmpty(val)) {
-				ret.add(val);
-			}
-		}
 		return ret;
 	}