Record if HBO stats is from CPP or JAVA

presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java

@@ -58,7 +58,8 @@ public static HistoricalPlanStatistics updatePlanStatistics(
             HistoricalPlanStatistics historicalPlanStatistics,
             List<PlanStatistics> inputTableStatistics,
             PlanStatistics current,
-            HistoryBasedOptimizationConfig config)
+            HistoryBasedOptimizationConfig config,
+            boolean isNativeExecution)
     {
         List<HistoricalPlanStatisticsEntry> lastRunsStatistics = historicalPlanStatistics.getLastRunsStatistics();
 
@@ -69,7 +70,7 @@ public static HistoricalPlanStatistics updatePlanStatistics(
             newLastRunsStatistics.remove(similarStatsIndex.get().intValue());
         }
 
-        newLastRunsStatistics.add(new HistoricalPlanStatisticsEntry(current, inputTableStatistics));
+        newLastRunsStatistics.add(new HistoricalPlanStatisticsEntry(current, inputTableStatistics, isNativeExecution ? HistoricalPlanStatisticsEntry.SourceType.CPP : HistoricalPlanStatisticsEntry.SourceType.JAVA));
         int maxLastRuns = inputTableStatistics.isEmpty() ? 1 : config.getMaxLastRunsHistory();
         if (newLastRunsStatistics.size() > maxLastRuns) {
             newLastRunsStatistics.remove(0);

presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsManager.java

@@ -19,6 +19,7 @@
 import com.facebook.presto.metadata.SessionPropertyManager;
 import com.facebook.presto.spi.statistics.EmptyPlanStatisticsProvider;
 import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider;
+import com.facebook.presto.sql.analyzer.FeaturesConfig;
 import com.facebook.presto.sql.planner.CachingPlanCanonicalInfoProvider;
 import com.facebook.presto.sql.planner.PlanCanonicalInfoProvider;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -39,16 +40,18 @@ public class HistoryBasedPlanStatisticsManager
 
     private HistoryBasedPlanStatisticsProvider historyBasedPlanStatisticsProvider = EmptyPlanStatisticsProvider.getInstance();
     private boolean statisticsProviderAdded;
+    private final boolean isNativeExecution;
 
     @Inject
-    public HistoryBasedPlanStatisticsManager(ObjectMapper objectMapper, SessionPropertyManager sessionPropertyManager, Metadata metadata, HistoryBasedOptimizationConfig config)
+    public HistoryBasedPlanStatisticsManager(ObjectMapper objectMapper, SessionPropertyManager sessionPropertyManager, Metadata metadata, HistoryBasedOptimizationConfig config, FeaturesConfig featuresConfig)
     {
         requireNonNull(objectMapper, "objectMapper is null");
         this.sessionPropertyManager = requireNonNull(sessionPropertyManager, "sessionPropertyManager is null");
         this.historyBasedStatisticsCacheManager = new HistoryBasedStatisticsCacheManager();
         ObjectMapper newObjectMapper = objectMapper.copy().configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true);
         this.planCanonicalInfoProvider = new CachingPlanCanonicalInfoProvider(historyBasedStatisticsCacheManager, newObjectMapper, metadata);
         this.config = requireNonNull(config, "config is null");
+        this.isNativeExecution = featuresConfig.isNativeExecutionEnabled();
     }
 
     public void addHistoryBasedPlanStatisticsProviderFactory(HistoryBasedPlanStatisticsProvider historyBasedPlanStatisticsProvider)
@@ -67,7 +70,7 @@ public HistoryBasedPlanStatisticsCalculator getHistoryBasedPlanStatisticsCalcula
 
     public HistoryBasedPlanStatisticsTracker getHistoryBasedPlanStatisticsTracker()
     {
-        return new HistoryBasedPlanStatisticsTracker(() -> historyBasedPlanStatisticsProvider, historyBasedStatisticsCacheManager, sessionPropertyManager, config);
+        return new HistoryBasedPlanStatisticsTracker(() -> historyBasedPlanStatisticsProvider, historyBasedStatisticsCacheManager, sessionPropertyManager, config, isNativeExecution);
     }
 
     public PlanCanonicalInfoProvider getPlanCanonicalInfoProvider()

presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java

@@ -78,17 +78,20 @@ public class HistoryBasedPlanStatisticsTracker
     private final HistoryBasedStatisticsCacheManager historyBasedStatisticsCacheManager;
     private final SessionPropertyManager sessionPropertyManager;
     private final HistoryBasedOptimizationConfig config;
+    private final boolean isNativeExecution;
 
     public HistoryBasedPlanStatisticsTracker(
             Supplier<HistoryBasedPlanStatisticsProvider> historyBasedPlanStatisticsProvider,
             HistoryBasedStatisticsCacheManager historyBasedStatisticsCacheManager,
             SessionPropertyManager sessionPropertyManager,
-            HistoryBasedOptimizationConfig config)
+            HistoryBasedOptimizationConfig config,
+            boolean isNativeExecution)
     {
         this.historyBasedPlanStatisticsProvider = requireNonNull(historyBasedPlanStatisticsProvider, "historyBasedPlanStatisticsProvider is null");
         this.historyBasedStatisticsCacheManager = requireNonNull(historyBasedStatisticsCacheManager, "historyBasedStatisticsCacheManager is null");
         this.sessionPropertyManager = requireNonNull(sessionPropertyManager, "sessionPropertyManager is null");
         this.config = requireNonNull(config, "config is null");
+        this.isNativeExecution = isNativeExecution;
     }
 
     public void updateStatistics(QueryExecution queryExecution)
@@ -328,7 +331,8 @@ public void updateStatistics(QueryInfo queryInfo)
                                     historicalPlanStatistics,
                                     historyBasedSourceInfo.getInputTableStatistics().get(),
                                     entry.getValue().getPlanStatistics(),
-                                    config);
+                                    config,
+                                    isNativeExecution);
                         }));
 
         if (!newPlanStatistics.isEmpty()) {

presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java

@@ -437,7 +437,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig,
         this.statsNormalizer = new StatsNormalizer();
         this.scalarStatsCalculator = new ScalarStatsCalculator(metadata);
         this.filterStatsCalculator = new FilterStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer);
-        this.historyBasedPlanStatisticsManager = new HistoryBasedPlanStatisticsManager(objectMapper, new SessionPropertyManager(), metadata, new HistoryBasedOptimizationConfig());
+        this.historyBasedPlanStatisticsManager = new HistoryBasedPlanStatisticsManager(objectMapper, new SessionPropertyManager(), metadata, new HistoryBasedOptimizationConfig(), featuresConfig);
         this.fragmentStatsProvider = new FragmentStatsProvider();
         this.statsCalculator = createNewStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer, filterStatsCalculator, historyBasedPlanStatisticsManager, fragmentStatsProvider);
         this.taskCountEstimator = new TaskCountEstimator(() -> nodeCountForStats);

presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java

@@ -97,7 +97,8 @@ private static HistoricalPlanStatistics updatePlanStatistics(
                 historicalPlanStatistics,
                 inputTableStatistics,
                 current,
-                new HistoryBasedOptimizationConfig());
+                new HistoryBasedOptimizationConfig(),
+                false);
     }
 
     private static PlanStatistics getPredictedPlanStatistics(

presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedStatsProvider.java

@@ -126,7 +126,7 @@ public Map<PlanNodeWithHash, HistoricalPlanStatistics> getStats(List<PlanNodeWit
                             if (node.getTable().toString().contains("orders")) {
                                 return new HistoricalPlanStatistics(ImmutableList.of(new HistoricalPlanStatisticsEntry(
                                         new PlanStatistics(Estimate.of(100), Estimate.of(1000), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()),
-                                        ImmutableList.of(new PlanStatistics(Estimate.of(15000), Estimate.unknown(), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty())))));
+                                        ImmutableList.of(new PlanStatistics(Estimate.of(15000), Estimate.unknown(), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty())), HistoricalPlanStatisticsEntry.SourceType.JAVA)));
                             }
                         }
                         return HistoricalPlanStatistics.empty();

presto-main/src/test/java/com/facebook/presto/dispatcher/TestLocalDispatchQuery.java

@@ -44,6 +44,7 @@
 import com.facebook.presto.spi.prerequisites.QueryPrerequisitesContext;
 import com.facebook.presto.spi.resourceGroups.ResourceGroupId;
 import com.facebook.presto.spi.security.AccessDeniedException;
+import com.facebook.presto.sql.analyzer.FeaturesConfig;
 import com.facebook.presto.transaction.TransactionManager;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableMap;
@@ -454,7 +455,7 @@ private QueryMonitor createQueryMonitor(CountingEventListener eventListener)
                 new SessionPropertyManager(),
                 metadata,
                 new QueryMonitorConfig(),
-                new HistoryBasedPlanStatisticsManager(new ObjectMapper(), new SessionPropertyManager(), metadata, new HistoryBasedOptimizationConfig()));
+                new HistoryBasedPlanStatisticsManager(new ObjectMapper(), new SessionPropertyManager(), metadata, new HistoryBasedOptimizationConfig(), new FeaturesConfig()));
     }
 
     private EventListenerManager createEventListenerManager(CountingEventListener countingEventListener)

presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestPrestoSparkStatsCalculator.java

@@ -129,7 +129,7 @@ public void testUsesHboStatsWhenMatchRuntime()
                         ImmutableList.of(
                                 new HistoricalPlanStatisticsEntry(
                                         new PlanStatistics(Estimate.of(100), Estimate.of(1000), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()),
-                                        ImmutableList.of())))));
+                                        ImmutableList.of(), HistoricalPlanStatisticsEntry.SourceType.JAVA)))));
 
         tester.assertStatsFor(pb -> pb.remoteSource(ImmutableList.of(new PlanFragmentId(1)), statsEquivalentRemoteSource))
                 .check(check -> check.totalSize(1000)
@@ -178,7 +178,7 @@ public void testUsesRuntimeStatsWhenHboDisabled()
                         ImmutableList.of(
                                 new HistoricalPlanStatisticsEntry(
                                         new PlanStatistics(Estimate.of(100), Estimate.of(1000), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()),
-                                        ImmutableList.of())))));
+                                        ImmutableList.of(), HistoricalPlanStatisticsEntry.SourceType.JAVA)))));
 
         tester.assertStatsFor(pb -> pb.remoteSource(ImmutableList.of(new PlanFragmentId(1))))
                 .check(check -> check.totalSize(1000)
@@ -205,7 +205,7 @@ public void testUsesRuntimeStatsWhenDiffersFromHbo()
                         ImmutableList.of(
                                 new HistoricalPlanStatisticsEntry(
                                         new PlanStatistics(Estimate.of(10), Estimate.of(100), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()),
-                                        ImmutableList.of())))));
+                                        ImmutableList.of(), HistoricalPlanStatisticsEntry.SourceType.JAVA)))));
 
         tester.assertStatsFor(pb -> pb.remoteSource(ImmutableList.of(new PlanFragmentId(1))))
                 .check(check -> check.totalSize(1000)

presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoricalPlanStatisticsEntry.java

@@ -14,6 +14,7 @@
 package com.facebook.presto.spi.statistics;
 
 import com.facebook.drift.annotations.ThriftConstructor;
+import com.facebook.drift.annotations.ThriftEnum;
 import com.facebook.drift.annotations.ThriftField;
 import com.facebook.drift.annotations.ThriftStruct;
 
@@ -28,16 +29,24 @@
 @ThriftStruct
 public class HistoricalPlanStatisticsEntry
 {
+    @ThriftEnum
+    public enum SourceType
+    {
+        JAVA, CPP;
+    }
+
     private final PlanStatistics planStatistics;
     // Size of input tables when plan statistics was recorded. This list will be sorted by input tables canonical order.
     private final List<PlanStatistics> inputTableStatistics;
+    private final SourceType sourceType;
 
     @ThriftConstructor
-    public HistoricalPlanStatisticsEntry(PlanStatistics planStatistics, List<PlanStatistics> inputTableStatistics)
+    public HistoricalPlanStatisticsEntry(PlanStatistics planStatistics, List<PlanStatistics> inputTableStatistics, SourceType sourceType)
     {
         // Check for nulls, to make it thrift backwards compatible
         this.planStatistics = planStatistics == null ? PlanStatistics.empty() : planStatistics;
         this.inputTableStatistics = unmodifiableList(inputTableStatistics == null ? emptyList() : inputTableStatistics);
+        this.sourceType = sourceType;
     }
 
     @ThriftField(value = 1, requiredness = OPTIONAL)
@@ -52,6 +61,12 @@ public List<PlanStatistics> getInputTableStatistics()
         return inputTableStatistics;
     }
 
+    @ThriftField(value = 3, requiredness = OPTIONAL)
+    public SourceType getSourceType()
+    {
+        return sourceType;
+    }
+
     @Override
     public boolean equals(Object o)
     {
@@ -62,18 +77,18 @@ public boolean equals(Object o)
             return false;
         }
         HistoricalPlanStatisticsEntry that = (HistoricalPlanStatisticsEntry) o;
-        return Objects.equals(planStatistics, that.planStatistics) && Objects.equals(inputTableStatistics, that.inputTableStatistics);
+        return Objects.equals(planStatistics, that.planStatistics) && Objects.equals(inputTableStatistics, that.inputTableStatistics) && Objects.equals(sourceType, that.sourceType);
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hash(planStatistics, inputTableStatistics);
+        return Objects.hash(planStatistics, inputTableStatistics, sourceType);
     }
 
     @Override
     public String toString()
     {
-        return format("HistoricalPlanStatisticsEntry{planStatistics=%s, inputTableStatistics=%s}", planStatistics, inputTableStatistics);
+        return format("HistoricalPlanStatisticsEntry{planStatistics=%s, inputTableStatistics=%s, sourceType=%s}", planStatistics, inputTableStatistics, sourceType);
     }
 }

redis-hbo-provider/src/test/java/com/facebook/presto/statistic/TestHistoricalStatisticsSerde.java

@@ -39,7 +39,7 @@ public void testSimpleHistoricalStatisticsEncoderDecoder()
     {
         HistoricalPlanStatistics samplePlanStatistics = new HistoricalPlanStatistics(ImmutableList.of(new HistoricalPlanStatisticsEntry(
                 new PlanStatistics(Estimate.of(100), Estimate.of(1000), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()),
-                ImmutableList.of(new PlanStatistics(Estimate.of(15000), Estimate.unknown(), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty())))));
+                ImmutableList.of(new PlanStatistics(Estimate.of(15000), Estimate.unknown(), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty())), HistoricalPlanStatisticsEntry.SourceType.JAVA)));
         HistoricalStatisticsSerde historicalStatisticsEncoderDecoder = new HistoricalStatisticsSerde();
 
         // Test PlanHash
@@ -57,7 +57,7 @@ public void testHistoricalPlanStatisticsEntryList()
         List<HistoricalPlanStatisticsEntry> historicalPlanStatisticsEntryList = new ArrayList<>();
         for (int i = 0; i < 50; i++) {
             historicalPlanStatisticsEntryList.add(new HistoricalPlanStatisticsEntry(new PlanStatistics(Estimate.of(i * 5), Estimate.of(i * 5), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()),
-                    ImmutableList.of(new PlanStatistics(Estimate.of(100), Estimate.of(i), 0, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()))));
+                    ImmutableList.of(new PlanStatistics(Estimate.of(100), Estimate.of(i), 0, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty())), HistoricalPlanStatisticsEntry.SourceType.JAVA));
         }
         HistoricalPlanStatistics samplePlanStatistics = new HistoricalPlanStatistics(historicalPlanStatisticsEntryList);
         HistoricalStatisticsSerde historicalStatisticsEncoderDecoder = new HistoricalStatisticsSerde();
@@ -88,7 +88,7 @@ public void testPlanStatisticsList()
         List<HistoricalPlanStatisticsEntry> historicalPlanStatisticsEntryList = new ArrayList<>();
         for (int i = 0; i < 50; i++) {
             historicalPlanStatisticsEntryList.add(new HistoricalPlanStatisticsEntry(new PlanStatistics(Estimate.of(i * 5), Estimate.of(i * 5), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()),
-                    planStatisticsEntryList));
+                    planStatisticsEntryList, HistoricalPlanStatisticsEntry.SourceType.JAVA));
         }
         HistoricalPlanStatistics samplePlanStatistics = new HistoricalPlanStatistics(historicalPlanStatisticsEntryList);
         HistoricalStatisticsSerde historicalStatisticsEncoderDecoder = new HistoricalStatisticsSerde();