diff --git a/api/src/main/java/org/open4goods/api/config/ApiConfig.java b/api/src/main/java/org/open4goods/api/config/ApiConfig.java index a47b0ccf..784e031c 100644 --- a/api/src/main/java/org/open4goods/api/config/ApiConfig.java +++ b/api/src/main/java/org/open4goods/api/config/ApiConfig.java @@ -119,8 +119,8 @@ SerialisationService serialisationService() { @Bean @Autowired - VerticalsGenerationService verticalsGenerationService(ProductRepository pRepo, SerialisationService serialisationService, AiService aiService, GoogleTaxonomyService gTaxoService) throws SAXException { - return new VerticalsGenerationService(apiProperties.getVerticalsGenerationConfig(), pRepo,serialisationService,aiService, gTaxoService); + VerticalsGenerationService verticalsGenerationService(ProductRepository pRepo, SerialisationService serialisationService, AiService aiService, GoogleTaxonomyService gTaxoService, VerticalsConfigService verticalsConfigService) throws SAXException { + return new VerticalsGenerationService(apiProperties.getVerticalsGenerationConfig(), pRepo,serialisationService,aiService, gTaxoService, verticalsConfigService); } diff --git a/api/src/main/java/org/open4goods/api/controller/api/VerticalsGenerationController.java b/api/src/main/java/org/open4goods/api/controller/api/VerticalsGenerationController.java index 5217dfb7..ba8f4403 100644 --- a/api/src/main/java/org/open4goods/api/controller/api/VerticalsGenerationController.java +++ b/api/src/main/java/org/open4goods/api/controller/api/VerticalsGenerationController.java @@ -5,16 +5,20 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Map.Entry; +import org.open4goods.api.model.VerticalAttributesStats; import org.open4goods.api.model.VerticalCategoryMapping; import org.open4goods.api.services.VerticalsGenerationService; import org.open4goods.commons.config.yml.ui.VerticalConfig; import org.open4goods.commons.exceptions.ResourceNotFoundException; +import org.open4goods.commons.model.constants.CacheConstants; import org.open4goods.commons.model.constants.RolesConstants; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.Cacheable; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import io.swagger.v3.oas.annotations.Operation; @@ -91,6 +95,7 @@ public void exportMapping() throws ResourceNotFoundException, IOException { } + @GetMapping(path="/mappings/generate/verticals") @Operation(summary="Generate verticals from the mapping") @PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')") @@ -99,4 +104,25 @@ public List generateVerticals() throws ResourceNotFoundException } + @GetMapping(path="/assist/attributes/{vertical}") + @Operation(summary="Generate attributes coverage for a vertical") + @PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')") +// @Cacheable(keyGenerator = CacheConstants.KEY_GENERATOR, cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public VerticalAttributesStats generateAttributesCoverage(@PathVariable String vertical) throws ResourceNotFoundException, IOException { + return verticalsGenService.attributesStats(vertical); + + } + + + @GetMapping(path="/assist/categories") + @Operation(summary="Generate the categories yaml fragment for a given match") + @PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')") +// @Cacheable(keyGenerator = CacheConstants.KEY_GENERATOR, cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public String generateCategoryMappingsFragment(@RequestParam String category) throws ResourceNotFoundException, IOException { + return verticalsGenService.generateCategoryMappingFragmentFor(category); + + } + + + } diff --git a/api/src/main/java/org/open4goods/api/model/AttributesStats.java b/api/src/main/java/org/open4goods/api/model/AttributesStats.java new file mode 100644 index 00000000..77e5f48a --- /dev/null +++ b/api/src/main/java/org/open4goods/api/model/AttributesStats.java @@ -0,0 +1,66 @@ +package org.open4goods.api.model; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.open4goods.commons.model.product.ProductAttribute; + +public class AttributesStats { + + /** + * Number of hits for this attribute + */ + private Integer hits = 0; + + /** + * The values, with associated popularity + */ + private Map values = new LinkedHashMap<>(); + + public Map getValues() { + return values; + } + + public void setValues(Map values) { + this.values = values; + } + + /** + * Increments the stats + * @param key the key for the attribute + * @param value the ProductAttribute object + */ + public void process(String key, ProductAttribute value) { + hits++; + + // Incrementing stats by value (value.getValue()) + // Assuming value.getValue() returns a string that represents the attribute value. + if (value != null && value.getValue() != null) { + values.compute(value.getValue(), (k, v) -> v == null ? 1 : v + 1); + } + } + + public Integer getHits() { + return hits; + } + + public void setHits(Integer hits) { + this.hits = hits; + } + + /** + * Sorts the values map by integer value in descending order. + */ + public void sort() { + values = values.entrySet() + .stream() + .sorted((e1, e2) -> Integer.compare(e2.getValue(), e1.getValue())) // Descending order + .collect(LinkedHashMap::new, // Collect into a LinkedHashMap to maintain order + (map, entry) -> map.put(entry.getKey(), entry.getValue()), + LinkedHashMap::putAll); + } + + + + +} diff --git a/api/src/main/java/org/open4goods/api/model/VerticalAttributesStats.java b/api/src/main/java/org/open4goods/api/model/VerticalAttributesStats.java new file mode 100644 index 00000000..3bd8d3c8 --- /dev/null +++ b/api/src/main/java/org/open4goods/api/model/VerticalAttributesStats.java @@ -0,0 +1,108 @@ +package org.open4goods.api.model; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; + +import org.open4goods.commons.model.product.ProductAttribute; + +/** + * Repesents attributes stats for a vertical + */ +public class VerticalAttributesStats { + + private Integer totalItems = 0; + + /** + * Stats by attribute name + */ + private Map stats = new LinkedHashMap<>(); + + public Integer getTotalItems() { + return totalItems; + } + + public void setTotalItems(Integer totalItems) { + this.totalItems = totalItems; + } + + public Map getStats() { + return stats; + } + + public void setStats(Map stats) { + this.stats = stats; + } + + /** + * Increments the stats with provided attributes + * @param all + */ + public void process(Map attrs) { + totalItems++; + + for (Entry attrEntry : attrs.entrySet()) { + AttributesStats as = stats.get(attrEntry.getKey()); + + if (null == as) { + as = new AttributesStats(); + } + + as.process(attrEntry.getKey(), attrEntry.getValue()); + stats.put(attrEntry.getKey(), as); + + } + } + + /** + * Sort the data for better restitution + */ + public void sort() { + + // Sorting the attribute names + stats = stats.entrySet() + .stream() + .sorted((e1, e2) -> Integer.compare(e2.getValue().getHits(), e1.getValue().getHits())) // Descending order + .collect(LinkedHashMap::new, // Collect into a LinkedHashMap to maintain order + (map, entry) -> map.put(entry.getKey(), entry.getValue()), + LinkedHashMap::putAll); + + // Sorting the attributes values frequency + stats.values().forEach(e -> { + e.sort(); + }); + + + } + + /** + * Cleaning by evicting dummy / noisy values + */ + public void clean() { + // Removing items where hits = number of attributes (means 1 to 1, like descriptif, title,gtin...) + Set toRemove = stats.entrySet().stream().filter(e->e.getValue().getHits() == e.getValue().getValues().size() ).map(e->e.getKey()) + .collect(Collectors.toSet()); + + + // Removing if only low keys + stats.entrySet().stream().forEach(e-> { + Integer max = e.getValue().getValues().values().stream().max(Integer::compare).orElse(0); + // TODO(p3,conf) : From conf + if (max < 5) { + toRemove.add(e.getKey()); + } + }); + + toRemove.forEach(e-> { + stats.remove(e); + }); + + } + + + + +} diff --git a/api/src/main/java/org/open4goods/api/services/VerticalsGenerationService.java b/api/src/main/java/org/open4goods/api/services/VerticalsGenerationService.java index cecb261c..bc8f0139 100644 --- a/api/src/main/java/org/open4goods/api/services/VerticalsGenerationService.java +++ b/api/src/main/java/org/open4goods/api/services/VerticalsGenerationService.java @@ -17,6 +17,7 @@ import org.apache.commons.lang3.StringUtils; import org.open4goods.api.config.yml.VerticalsGenerationConfig; +import org.open4goods.api.model.VerticalAttributesStats; import org.open4goods.api.model.VerticalCategoryMapping; import org.open4goods.commons.config.yml.ui.ProductI18nElements; import org.open4goods.commons.config.yml.ui.VerticalConfig; @@ -25,6 +26,7 @@ import org.open4goods.commons.model.product.Product; import org.open4goods.commons.services.GoogleTaxonomyService; import org.open4goods.commons.services.SerialisationService; +import org.open4goods.commons.services.VerticalsConfigService; import org.open4goods.commons.services.ai.AiService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +37,7 @@ public class VerticalsGenerationService { private static final Logger LOGGER = LoggerFactory.getLogger(VerticalsGenerationService.class); private VerticalsGenerationConfig config; + private VerticalsConfigService verticalConfigservice; private ProductRepository repository; private SerialisationService serialisationService; @@ -44,13 +47,14 @@ public class VerticalsGenerationService { private AiService aiService; private GoogleTaxonomyService googleTaxonomyService; - public VerticalsGenerationService(VerticalsGenerationConfig config, ProductRepository repository, SerialisationService serialisationService, AiService aiService, GoogleTaxonomyService googleTaxonomyService) { + public VerticalsGenerationService(VerticalsGenerationConfig config, ProductRepository repository, SerialisationService serialisationService, AiService aiService, GoogleTaxonomyService googleTaxonomyService, VerticalsConfigService verticalsConfigService) { super(); this.config = config; this.repository = repository; this.serialisationService = serialisationService; this.aiService = aiService; this.googleTaxonomyService = googleTaxonomyService; + this.verticalConfigservice = verticalsConfigService; } @@ -302,6 +306,31 @@ public List generateVerticals() { + /** + * Compute the attributes coverage stats for this vertical + * @param vertical + * @return + */ + public VerticalAttributesStats attributesStats(String vertical) { + VerticalConfig vc = verticalConfigservice.getConfigById(vertical); + VerticalAttributesStats ret = new VerticalAttributesStats() ; + if (null != vc) { + LOGGER.info("Attributes stats for vertical {} is running",vertical); + repository.exportVerticalWithValidDate(vc, false).forEach(p -> { + ret.process(p.getAttributes().getAll()); + }); + + // Cleaning the values + ret.clean(); + + // Sorting the values + ret.sort(); + } + + return ret; + } + + /** * Generate a vertical stub, using our matching categories detected and adding informations through AI * @param cat @@ -387,6 +416,27 @@ private Integer resolveGoogleTaxonomy(String string) { return ret; } + + + /** + * Generate the yaml fragment for a given category match + * @param category + * @return + */ + public String generateCategoryMappingFragmentFor(String category) { + + VerticalCategoryMapping mapping = sortedMappings.get(category); + + StringBuilder ret = new StringBuilder(); + ret.append("matchingCategories:").append("\n"); + ret.append(" - \"").append(category).append("\"\n"); + for (String cat : mapping.getAssociatedCategories().keySet()) { + ret.append(" - \"").append(cat).append("\"\n"); + } + + return ret.toString(); + } + }