reduce logging frequency for skipped records

noi-techpark · Jan 21, 2025 · 6426261 · 6426261
1 parent fe75c80
commit 6426261
Showing 1 changed file with 23 additions and 16 deletions.
diff --git a/...r/src/main/java/com/opendatahub/timeseries/bdp/writer/dal/MeasurementAbstractHistory.java b/...r/src/main/java/com/opendatahub/timeseries/bdp/writer/dal/MeasurementAbstractHistory.java
@@ -9,31 +9,28 @@
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.Map.Entry;
-import java.util.function.BiFunction;
 import java.util.function.Function;
 
-import jakarta.persistence.CascadeType;
-import jakarta.persistence.Column;
-import jakarta.persistence.EntityManager;
-import jakarta.persistence.FetchType;
-import jakarta.persistence.ManyToOne;
-import jakarta.persistence.MappedSuperclass;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.opendatahub.timeseries.bdp.writer.dal.util.JPAException;
-import com.opendatahub.timeseries.bdp.writer.dal.util.Log;
-import com.opendatahub.timeseries.bdp.writer.dal.util.QueryBuilder;
 import com.opendatahub.timeseries.bdp.dto.dto.DataMapDto;
 import com.opendatahub.timeseries.bdp.dto.dto.RecordDto;
 import com.opendatahub.timeseries.bdp.dto.dto.RecordDtoImpl;
 import com.opendatahub.timeseries.bdp.dto.dto.SimpleRecordDto;
+import com.opendatahub.timeseries.bdp.writer.dal.util.JPAException;
+import com.opendatahub.timeseries.bdp.writer.dal.util.Log;
+import com.opendatahub.timeseries.bdp.writer.dal.util.QueryBuilder;
+
+import jakarta.persistence.CascadeType;
+import jakarta.persistence.Column;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.FetchType;
+import jakarta.persistence.ManyToOne;
+import jakarta.persistence.MappedSuperclass;
 
 /**
  * <p>
@@ -248,6 +245,7 @@ public static void pushRecords(EntityManager em, String stationType, DataMapDto<
                             }
                         }
 
+                        int skippedRecs = 0;
                         for (Period period : periods.values()) {
                             period.number.updateLatest(em, (newest) -> {
                                 return new Measurement(station, type, ((Number) newest.getValue()).doubleValue(),
@@ -263,10 +261,17 @@ public static void pushRecords(EntityManager em, String stationType, DataMapDto<
                                         new Date(newest.getTimestamp()),
                                         period.period);
                             });
+
+                            skippedRecs = skippedRecs + period.skippedRecs;
                         }
-
+
+                        if (skippedRecs > 0) {
+                            log.warn(String.format("Skipped %d records due to timestamp for type: [%s, %s, %s]",
+                                    skippedRecs,
+                                    station.stationtype, station.stationcode, type.getCname()));
+                        }
+
                         em.getTransaction().commit();
-
                     } catch (Exception ex) {
                         log.error(
                                 String.format("Exception '%s'... Skipping this measurement!", ex.getMessage()),
@@ -298,6 +303,7 @@ private static class Period {
         private DataType type;
         private Integer period;
         private Provenance provenance;
+        public int skippedRecs = 0;
 
         private class TimeSeries {
             private MeasurementAbstract latest;
@@ -324,8 +330,9 @@ public void addHistory(EntityManager em, Log log, SimpleRecordDto dto, Measureme
                     em.persist(rec);
                     updateNewest(dto);
                 } else {
-                    log.warn(String.format("Skipping record due to timestamp: [%s, %s, %s, %d, %d]",
+                    LOG.debug(String.format("Skipping record due to timestamp: [%s, %s, %s, %d, %d]",
                             station.stationtype, station.stationcode, type.getCname(), period, dto.getTimestamp()));
+                    skippedRecs++;
                 }
             }