Skip to content

Commit

Permalink
Code formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
@lbschanno
lbschanno committed Sep 3, 2024
1 parent 8933a7e commit 8cf9be7
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 27 deletions.
2 changes: 1 addition & 1 deletion src/main/java/datawave/iterators/FrequencyMetadataAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import java.util.Set;
import java.util.TreeMap;

import datawave.util.time.DateHelper;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
Expand All @@ -33,6 +32,7 @@
import datawave.marking.MarkingFunctions;
import datawave.query.model.DateFrequencyMap;
import datawave.util.StringUtils;
import datawave.util.time.DateHelper;

/**
* Aggregates entries in the metadata table for the "f", "i", and "ri" columns. When initially ingested, entries for these columns have a column qualifier with
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/datawave/query/util/AllFieldMetadataHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.util.TreeSet;
import java.util.concurrent.ExecutionException;

import datawave.iterators.FrequencyMetadataAggregator;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
Expand Down Expand Up @@ -60,6 +59,7 @@
import datawave.data.ColumnFamilyConstants;
import datawave.data.type.Type;
import datawave.data.type.TypeFactory;
import datawave.iterators.FrequencyMetadataAggregator;
import datawave.query.composite.CompositeMetadata;
import datawave.query.composite.CompositeMetadataHelper;
import datawave.query.model.DateFrequencyMap;
Expand Down Expand Up @@ -857,7 +857,8 @@ protected HashMap<String,Long> getCountsByFieldInDayWithTypes(Entry<String,Strin
scanner.fetchColumnFamily(ColumnFamilyConstants.COLF_F);
scanner.setRange(Range.exact(fieldName));

// It's possible to find rows with column qualifiers in the format <datatype> (aggregated entries) and/or <datatype>\0<date> (non-aggregated entries).
// It's possible to find rows with column qualifiers in the format <datatype> (aggregated entries) and/or <datatype>\0<date> (non-aggregated
// entries).
// Filter out any non-aggregated entries that does not have the date in the column qualifier.
IteratorSetting cqRegex = new IteratorSetting(50, RegExFilter.class);
// Allow any entries that do not contain the null byte delimiter, or contain it with the target date directly afterwards.
Expand Down Expand Up @@ -1638,7 +1639,6 @@ private void addToTargetMap(String datatype, DateFrequencyMap aggregatedCounts)
}
}


/**
* Add the current date and count to the current target map for the given datatype.
*/
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/datawave/query/util/MetadataHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import datawave.iterators.FrequencyMetadataAggregator;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
Expand Down Expand Up @@ -74,6 +73,7 @@
import datawave.data.MetadataCardinalityCounts;
import datawave.data.type.Type;
import datawave.iterators.EdgeMetadataCombiner;
import datawave.iterators.FrequencyMetadataAggregator;
import datawave.iterators.MetadataFColumnSeekingFilter;
import datawave.iterators.filter.EdgeMetadataCQStrippingIterator;
import datawave.marking.MarkingFunctions;
Expand Down Expand Up @@ -1754,7 +1754,8 @@ public Date getEarliestOccurrenceOfFieldWithType(String fieldName, final String
* a wrapped AccumuloClient
* @return the earliest date the field is found, or null otherwise
*/
protected Date getEarliestOccurrenceOfFieldWithType(String fieldName, final String datatypeFilter, AccumuloClient client, WrappedAccumuloClient wrappedClient) {
protected Date getEarliestOccurrenceOfFieldWithType(String fieldName, final String datatypeFilter, AccumuloClient client,
WrappedAccumuloClient wrappedClient) {
String earliestDate = null;
String prevDatatype = null;
boolean skipToAggregated = false;
Expand Down
54 changes: 34 additions & 20 deletions src/test/java/datawave/iterators/FrequencyMetadataAggregatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,8 @@ public void testCombiningColumnVisibilities() throws TableNotFoundException {
// Enable to option to combine visibilities.
givenCombineColumnVisibilitiesIsTrue();

expect("NAME", COLF_F, "csv" + NULL_BYTE + "AGGREGATED", "BAR&COB&FOO", 1500000015L, createDateFrequencyMap("20200101", 16L, "20200102", 26L, "20200103", 24L, "20200104", 16L));
expect("NAME", COLF_F, "csv" + NULL_BYTE + "AGGREGATED", "BAR&COB&FOO", 1500000015L,
createDateFrequencyMap("20200101", 16L, "20200102", 26L, "20200103", 24L, "20200104", 16L));

assertResults();
}
Expand Down Expand Up @@ -327,7 +328,8 @@ void testAggregatedAndNonAggregatedEntries() throws TableNotFoundException {
givenNonAggregatedRow("NAME", COLF_F, "csv", "FOO", 1500000002L, "20200103", 3L);
givenNonAggregatedRow("NAME", COLF_F, "csv", "FOO", 1500000003L, "20200103", 3L);

expect("NAME", COLF_F, "csv" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 40L, "20200101", 19L, "20200102", 30L, "20200103", 12L));
expect("NAME", COLF_F, "csv" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L,
createDateFrequencyMap("20191225", 40L, "20200101", 19L, "20200102", 30L, "20200103", 12L));

assertResults();
}
Expand All @@ -353,19 +355,26 @@ void testNoAggregationNeeded() throws TableNotFoundException {
givenAggregatedRow("GENDER", COLF_I, "attr", "FOO", 1499999995L, createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
givenAggregatedRow("GENDER", COLF_RI, "attr", "FOO", 1499999995L, createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));

expect("GENDER", COLF_F, "attr" + NULL_BYTE + "AGGREGATED", "BAR", 1499999995L, createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_F, "attr" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_I, "attr" + NULL_BYTE + "AGGREGATED", "BAR", 1499999995L, createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_I, "attr" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_RI, "attr" + NULL_BYTE + "AGGREGATED", "BAR", 1499999995L, createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_RI, "attr" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_F, "attr" + NULL_BYTE + "AGGREGATED", "BAR", 1499999995L,
createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_F, "attr" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L,
createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_I, "attr" + NULL_BYTE + "AGGREGATED", "BAR", 1499999995L,
createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_I, "attr" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L,
createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_RI, "attr" + NULL_BYTE + "AGGREGATED", "BAR", 1499999995L,
createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("GENDER", COLF_RI, "attr" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L,
createDateFrequencyMap("20191220", 20L, "20191225", 10L, "20191230", 11L));
expect("NAME", COLF_F, "csv" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20191225", 40L, "20200101", 15L, "20200102", 20L));
expect("NAME", COLF_F, "text" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20200101", 20L, "20200102", 10L));
expect("NAME", COLF_F, "wiki" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20191225", 20L, "20200101", 10L));
expect("NAME", COLF_I, "csv" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20191225", 40L, "20200101", 15L, "20200102", 20L));
expect("NAME", COLF_I, "text" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20200101", 20L, "20200102", 10L));
expect("NAME", COLF_I, "wiki" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20191225", 20L, "20200101", 10L));
expect("NAME", COLF_RI, "csv" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20191225", 40L, "20200101", 15L, "20200102", 20L));
expect("NAME", COLF_RI, "csv" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L,
createDateFrequencyMap("20191225", 40L, "20200101", 15L, "20200102", 20L));
expect("NAME", COLF_RI, "text" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20200101", 20L, "20200102", 10L));
expect("NAME", COLF_RI, "wiki" + NULL_BYTE + "AGGREGATED", "FOO", 1499999995L, createDateFrequencyMap("20191225", 20L, "20200101", 10L));

Expand Down Expand Up @@ -396,10 +405,12 @@ void testDiverseDataset() throws TableNotFoundException {
givenNonAggregatedRow("JOB", COLF_F, "attr", "FOO", 1500000004L, "20200101", 1L); // Should result in new aggregated entry because new row.
givenNonAggregatedRow("JOB", COLF_I, "attr", "FOO", 1500000004L, "20200101", 1L); // Should result in new aggregated entry because new row.

expect("AGE", COLF_F, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L,
createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "num" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "var" + NULL_BYTE + "AGGREGATED", "BAR", 1500000004L, createDateFrequencyMap("20200101", 1L));
expect("AGE", COLF_I, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_I, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L,
createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_I, "num" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("GENDER", COLF_F, "text" + NULL_BYTE + "AGGREGATED", "BAR", 1499999999L, createDateFrequencyMap("20200101", 1L, "20200102", 1L));
expect("GENDER", COLF_F, "text" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20200101", 1L));
Expand Down Expand Up @@ -440,11 +451,12 @@ void testIndexMarkers() throws TableNotFoundException {
givenMutation("JOB", COLF_RI, "attr" + NULL_BYTE + "20190530" + NULL_BYTE + "false", "FOO", 1500000004L, new Value());
givenMutation("NAME", COLF_I, "attr" + NULL_BYTE + "20171201" + NULL_BYTE + "true", "BAR", 1500000004L, new Value());


expect("AGE", COLF_F, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L,
createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "num" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "var" + NULL_BYTE + "AGGREGATED", "BAR", 1500000004L, createDateFrequencyMap("20200101", 1L));
expect("AGE", COLF_I, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_I, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L,
createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_I, "num" + NULL_BYTE + "20191230" + NULL_BYTE + "true", "BAR", 1400000005L, new Value());
expect("AGE", COLF_I, "num" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("GENDER", COLF_F, "text" + NULL_BYTE + "AGGREGATED", "BAR", 1499999999L, createDateFrequencyMap("20200101", 1L, "20200102", 1L));
Expand Down Expand Up @@ -487,11 +499,12 @@ void testLegacyFormats() throws TableNotFoundException {
givenMutation("AGE", COLF_I, "num", "BAR", 1400000005L, new Value());
givenMutation("JOB", COLF_RI, "attr" + NULL_BYTE + "FakeTypeClassName", "FOO", 1500000004L, new Value());


expect("AGE", COLF_F, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L,
createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "num" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "var" + NULL_BYTE + "AGGREGATED", "BAR", 1500000004L, createDateFrequencyMap("20200101", 1L));
expect("AGE", COLF_I, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_I, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L,
createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_I, "num", "BAR", 1400000005L, new Value());
expect("AGE", COLF_I, "num" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("GENDER", COLF_F, "text" + NULL_BYTE + "AGGREGATED", "BAR", 1499999999L, createDateFrequencyMap("20200101", 1L, "20200102", 1L));
Expand All @@ -505,7 +518,6 @@ void testLegacyFormats() throws TableNotFoundException {
assertResults();
}


/**
* Verify that scanning over a table with columns that are not to be aggregated result in them being unchanged.
*/
Expand Down Expand Up @@ -546,10 +558,12 @@ void testMixedColumns() throws TableNotFoundException {
expect("AGE", COLF_DESC, "var", "BAR", 1400000005L, new Value("age_var description"));
expect("AGE", COLF_E, "lifetime", "BAR", 1400000005L, new Value());
expect("AGE", COLF_E, "num", "BAR", 1400000005L, new Value());
expect("AGE", COLF_F, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L,
createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "num" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_F, "var" + NULL_BYTE + "AGGREGATED", "BAR", 1500000004L, createDateFrequencyMap("20200101", 1L));
expect("AGE", COLF_I, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_I, "lifetime" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L,
createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("AGE", COLF_I, "num" + NULL_BYTE + "AGGREGATED", "FOO", 1500000004L, createDateFrequencyMap("20191225", 1L, "20200101", 2L, "20200102", 1L));
expect("GENDER", COLF_DESC, "text", "BAR", 1400000005L, new Value("gender_text description"));
expect("GENDER", COLF_F, "text" + NULL_BYTE + "AGGREGATED", "BAR", 1499999999L, createDateFrequencyMap("20200101", 1L, "20200102", 1L));
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/datawave/query/util/MetadataHelperTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import java.util.Objects;
import java.util.Set;

import datawave.iterators.FrequencyMetadataAggregator;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
Expand All @@ -39,6 +38,7 @@

import datawave.accumulo.inmemory.InMemoryAccumuloClient;
import datawave.accumulo.inmemory.InMemoryInstance;
import datawave.iterators.FrequencyMetadataAggregator;
import datawave.query.composite.CompositeMetadataHelper;
import datawave.query.model.DateFrequencyMap;
import datawave.util.time.DateHelper;
Expand Down

0 comments on commit 8cf9be7

Please sign in to comment.