diff --git a/presto-array/src/main/java/io/prestosql/array/BlockBigArray.java b/presto-array/src/main/java/io/prestosql/array/BlockBigArray.java index e05880add0093ea63cbeac59a9a9d443a3f40abf..3858068a605f2a65f630bea22821a6fceb3328be 100644 --- a/presto-array/src/main/java/io/prestosql/array/BlockBigArray.java +++ b/presto-array/src/main/java/io/prestosql/array/BlockBigArray.java @@ -96,6 +96,30 @@ public final class BlockBigArray array.set(index, value); } + /** + * Resets the element of this big array at specified index. + * + * @param index a position in this big array. + */ + public void reset(long index) + { + Block currentValue = array.get(index); + if (currentValue != null) { + currentValue.retainedBytesForEachPart((object, size) -> { + if (currentValue == object) { + // track instance size separately as the reference count for an instance is always 1 + sizeOfBlocks -= size; + return; + } + if (trackedObjects.decrementAndGet(object) == 0) { + // decrement the size only when it is the last reference + sizeOfBlocks -= size; + } + }); + } + array.reset(index); + } + /** * Ensures this big array is at least the specified length. If the array is smaller, segments * are added until the array is larger then the specified length. diff --git a/presto-array/src/main/java/io/prestosql/array/BooleanBigArray.java b/presto-array/src/main/java/io/prestosql/array/BooleanBigArray.java index 56f7d4fe818c71192e0661ee2c2cdc03ec16e34c..835bad5d4c2d0eba524d5c4d5b51c83fe5674b6a 100644 --- a/presto-array/src/main/java/io/prestosql/array/BooleanBigArray.java +++ b/presto-array/src/main/java/io/prestosql/array/BooleanBigArray.java @@ -81,6 +81,16 @@ public final class BooleanBigArray array[BigArrays.segment(index)][BigArrays.offset(index)] = value; } + /** + * Resets the element of this big array at specified index. + * + * @param index a position in this big array. + */ + public void reset(long index) + { + array[BigArrays.segment(index)][BigArrays.offset(index)] = initialValue; + } + /** * Ensures this big array is at least the specified length. If the array is smaller, segments * are added until the array is larger then the specified length. diff --git a/presto-array/src/main/java/io/prestosql/array/ByteBigArray.java b/presto-array/src/main/java/io/prestosql/array/ByteBigArray.java index 2ea227e95fed27554c1aaa960f8fe09b73283859..b3b8c3b9afafdbd739d98472f00a2250258d11da 100644 --- a/presto-array/src/main/java/io/prestosql/array/ByteBigArray.java +++ b/presto-array/src/main/java/io/prestosql/array/ByteBigArray.java @@ -81,6 +81,16 @@ public final class ByteBigArray array[BigArrays.segment(index)][BigArrays.offset(index)] = value; } + /** + * Resets the element of this big array at specified index. + * + * @param index a position in this big array. + */ + public void reset(long index) + { + array[BigArrays.segment(index)][BigArrays.offset(index)] = initialValue; + } + /** * Ensures this big array is at least the specified length. If the array is smaller, segments * are added until the array is larger then the specified length. diff --git a/presto-array/src/main/java/io/prestosql/array/DoubleBigArray.java b/presto-array/src/main/java/io/prestosql/array/DoubleBigArray.java index 9929cf4a6fc2559de9e2390c61ef8f4fc8981b82..410be0875a76a7b40b535a0b82ee0b11fc8a541d 100644 --- a/presto-array/src/main/java/io/prestosql/array/DoubleBigArray.java +++ b/presto-array/src/main/java/io/prestosql/array/DoubleBigArray.java @@ -84,6 +84,16 @@ public final class DoubleBigArray array[BigArrays.segment(index)][BigArrays.offset(index)] = value; } + /** + * Resets the element of this big array at specified index. + * + * @param index a position in this big array. + */ + public void reset(long index) + { + array[BigArrays.segment(index)][BigArrays.offset(index)] = initialValue; + } + /** * Adds the specified value to the specified element of this big array. * diff --git a/presto-array/src/main/java/io/prestosql/array/IntBigArray.java b/presto-array/src/main/java/io/prestosql/array/IntBigArray.java index 5d5c52f88dac03e388f573afd3e84f7e6379784f..c18d9d3d74d56d5bdca70628c23228e8f55e4d52 100644 --- a/presto-array/src/main/java/io/prestosql/array/IntBigArray.java +++ b/presto-array/src/main/java/io/prestosql/array/IntBigArray.java @@ -89,6 +89,16 @@ public final class IntBigArray array[BigArrays.segment(index)][BigArrays.offset(index)] = value; } + /** + * Resets the element of this big array at specified index. + * + * @param index a position in this big array. + */ + public void reset(long index) + { + array[BigArrays.segment(index)][BigArrays.offset(index)] = initialValue; + } + /** * Increments the element of this big array at specified index. * diff --git a/presto-array/src/main/java/io/prestosql/array/LongBigArray.java b/presto-array/src/main/java/io/prestosql/array/LongBigArray.java index 3dc64e38a51c216e7bb8c9703132aaab4574a973..dee5c57f9087c64552ba15d2255e07973c65b828 100644 --- a/presto-array/src/main/java/io/prestosql/array/LongBigArray.java +++ b/presto-array/src/main/java/io/prestosql/array/LongBigArray.java @@ -96,6 +96,16 @@ public final class LongBigArray array[segment(index)][offset(index)] = value; } + /** + * Resets the element of this big array at specified index. + * + * @param index a position in this big array. + */ + public void reset(long index) + { + array[segment(index)][offset(index)] = initialValue; + } + /** * Increments the element of this big array at specified index. * diff --git a/presto-array/src/main/java/io/prestosql/array/ObjectBigArray.java b/presto-array/src/main/java/io/prestosql/array/ObjectBigArray.java index e4128dbe7fa0ca941f78655eec3edca514f8cf1e..75ab52b1197b46cd65a16bd426ec6a1e290907d6 100644 --- a/presto-array/src/main/java/io/prestosql/array/ObjectBigArray.java +++ b/presto-array/src/main/java/io/prestosql/array/ObjectBigArray.java @@ -105,6 +105,17 @@ public final class ObjectBigArray array[BigArrays.segment(index)][offset(index)] = value; } + /** + * Resets the element of this big array at specified index. + * + * @param index a position in this big array. + * @return true if the previous value was null + */ + public void reset(long index) + { + array[BigArrays.segment(index)][offset(index)] = initialValue; + } + /** * Replaces the element of this big array at specified index. * diff --git a/presto-array/src/main/java/io/prestosql/array/ShortBigArray.java b/presto-array/src/main/java/io/prestosql/array/ShortBigArray.java index bcb682e80919b31ba7e17d637d3e51fc8768b063..398cd7376d19441bdf8feaf099dd585af4b93047 100644 --- a/presto-array/src/main/java/io/prestosql/array/ShortBigArray.java +++ b/presto-array/src/main/java/io/prestosql/array/ShortBigArray.java @@ -84,6 +84,16 @@ public final class ShortBigArray array[BigArrays.segment(index)][BigArrays.offset(index)] = value; } + /** + * Resets the element of this big array at specified index. + * + * @param index a position in this big array. + */ + public void reset(long index) + { + array[BigArrays.segment(index)][BigArrays.offset(index)] = initialValue; + } + /** * Increments the element of this big array at specified index. * diff --git a/presto-array/src/main/java/io/prestosql/array/SliceBigArray.java b/presto-array/src/main/java/io/prestosql/array/SliceBigArray.java index 3a6c97a332ae9276db3fb63e2a7c89b7e0bf18f8..bcd534b834c8f72e7bfb2bd0ba7810bde74084eb 100644 --- a/presto-array/src/main/java/io/prestosql/array/SliceBigArray.java +++ b/presto-array/src/main/java/io/prestosql/array/SliceBigArray.java @@ -73,6 +73,29 @@ public final class SliceBigArray array.set(index, value); } + /** + * Resets the element of this big array at specified index. + * + * @param index a position in this big array. + */ + public void reset(long index) + { + Slice currentValue = array.get(index); + if (currentValue != null) { + int baseReferenceCount = trackedSlices.decrementAndGet(currentValue.getBase()); + int sliceReferenceCount = trackedSlices.decrementAndGet(currentValue); + if (baseReferenceCount == 0) { + // it is the last referenced base + sizeOfSlices -= currentValue.getRetainedSize(); + } + else if (sliceReferenceCount == 0) { + // it is the last referenced slice + sizeOfSlices -= SLICE_INSTANCE_SIZE; + } + } + array.reset(index); + } + /** * Ensures this big array is at least the specified length. If the array is smaller, segments * are added until the array is larger then the specified length. diff --git a/presto-benchto-benchmarks/src/test/java/io/prestosql/sql/planner/AbstractCostBasedPlanTest.java b/presto-benchto-benchmarks/src/test/java/io/prestosql/sql/planner/AbstractCostBasedPlanTest.java index 796ff4cea90fb39299893890a1408ed189fe2991..31d082b0a7392676bd9d70c76fdf57ea92ccc267 100644 --- a/presto-benchto-benchmarks/src/test/java/io/prestosql/sql/planner/AbstractCostBasedPlanTest.java +++ b/presto-benchto-benchmarks/src/test/java/io/prestosql/sql/planner/AbstractCostBasedPlanTest.java @@ -31,6 +31,7 @@ import io.prestosql.spi.plan.AggregationNode; import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.TableScanNode; import io.prestosql.spi.plan.ValuesNode; import io.prestosql.sql.planner.assertions.BasePlanTest; @@ -282,6 +283,15 @@ public abstract class AbstractCostBasedPlanTest return visitPlan(node, indent + 1); } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Integer indent) + { + JoinNode.DistributionType distributionType = node.getDistributionType() + .orElseThrow(() -> new VerifyException("Expected distribution type to be set")); + output(indent, "join (%s, %s):", node.getType(), distributionType); + return visitPlan(node, indent + 1); + } + @Override public Void visitExchange(ExchangeNode node, Integer indent) { diff --git a/presto-main/src/main/java/io/prestosql/SystemSessionProperties.java b/presto-main/src/main/java/io/prestosql/SystemSessionProperties.java index 41d990d2f80dbb95ff7098eb7c9aee3fec60f98f..994de8ecb871e76d12875fbd7402f863fae9ea13 100644 --- a/presto-main/src/main/java/io/prestosql/SystemSessionProperties.java +++ b/presto-main/src/main/java/io/prestosql/SystemSessionProperties.java @@ -121,6 +121,7 @@ public final class SystemSessionProperties public static final String ENABLE_INTERMEDIATE_AGGREGATIONS = "enable_intermediate_aggregations"; public static final String PUSH_AGGREGATION_THROUGH_JOIN = "push_aggregation_through_join"; public static final String PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN = "push_partial_aggregation_through_join"; + public static final String MERGE_PARTIAL_AGGREGATION_WITH_JOIN = "merge_partial_aggregation_with_join"; public static final String PARSE_DECIMAL_LITERALS_AS_DOUBLE = "parse_decimal_literals_as_double"; public static final String FORCE_SINGLE_NODE_OUTPUT = "force_single_node_output"; public static final String FILTER_AND_PROJECT_MIN_OUTPUT_PAGE_SIZE = "filter_and_project_min_output_page_size"; @@ -627,6 +628,11 @@ public final class SystemSessionProperties "Push partial aggregations below joins", false, false), + booleanProperty( + MERGE_PARTIAL_AGGREGATION_WITH_JOIN, + "Merge partial aggregations with joins", + false, + false), booleanProperty( PARSE_DECIMAL_LITERALS_AS_DOUBLE, "Parse decimal literals as DOUBLE instead of DECIMAL", @@ -1452,6 +1458,11 @@ public final class SystemSessionProperties return session.getSystemProperty(PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN, Boolean.class); } + public static boolean isMergePartialAggregationWithJoin(Session session) + { + return session.getSystemProperty(MERGE_PARTIAL_AGGREGATION_WITH_JOIN, Boolean.class); + } + public static boolean isParseDecimalLiteralsAsDouble(Session session) { return session.getSystemProperty(PARSE_DECIMAL_LITERALS_AS_DOUBLE, Boolean.class); diff --git a/presto-main/src/main/java/io/prestosql/cost/CostCalculatorUsingExchanges.java b/presto-main/src/main/java/io/prestosql/cost/CostCalculatorUsingExchanges.java index 2af5f25c8044f63060fa45d9bbd174ae1f790a32..db32c0f4514b5ed64efc5ae9ccfe9b21c6bb191d 100644 --- a/presto-main/src/main/java/io/prestosql/cost/CostCalculatorUsingExchanges.java +++ b/presto-main/src/main/java/io/prestosql/cost/CostCalculatorUsingExchanges.java @@ -22,6 +22,7 @@ import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.GroupReference; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; @@ -247,6 +248,17 @@ public class CostCalculatorUsingExchanges return costForLookupJoin(node, localCost); } + @Override + public PlanCostEstimate visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + LocalCostEstimate localCost = calculateJoinCost( + node, + node.getLeft(), + node.getRight(), + Objects.equals(node.getDistributionType(), Optional.of(JoinNode.DistributionType.REPLICATED))); + return costForLookupJoin(node, localCost); + } + private LocalCostEstimate calculateJoinCost(PlanNode join, PlanNode probe, PlanNode build, boolean replicated) { int estimatedSourceDistributedTaskCount = taskCountEstimator.estimateSourceDistributedTaskCount(); diff --git a/presto-main/src/main/java/io/prestosql/cost/CostCalculatorWithEstimatedExchanges.java b/presto-main/src/main/java/io/prestosql/cost/CostCalculatorWithEstimatedExchanges.java index 9785c74c04814ebae63d8214c03321c88e2c6bd3..df4696b4410f855ec972eb340b7504a9c89b39fa 100644 --- a/presto-main/src/main/java/io/prestosql/cost/CostCalculatorWithEstimatedExchanges.java +++ b/presto-main/src/main/java/io/prestosql/cost/CostCalculatorWithEstimatedExchanges.java @@ -18,6 +18,7 @@ import io.prestosql.Session; import io.prestosql.spi.plan.AggregationNode; import io.prestosql.spi.plan.GroupReference; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.UnionNode; import io.prestosql.sql.planner.TypeProvider; @@ -135,6 +136,18 @@ public class CostCalculatorWithEstimatedExchanges taskCountEstimator.estimateSourceDistributedTaskCount()); } + @Override + public LocalCostEstimate visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + return calculateJoinExchangeCost( + node.getLeft(), + node.getRight(), + stats, + types, + Objects.equals(node.getDistributionType(), Optional.of(JoinNode.DistributionType.REPLICATED)), + taskCountEstimator.estimateSourceDistributedTaskCount()); + } + @Override public LocalCostEstimate visitSemiJoin(SemiJoinNode node, Void context) { diff --git a/presto-main/src/main/java/io/prestosql/cost/JoinOnAggregationStatsRule.java b/presto-main/src/main/java/io/prestosql/cost/JoinOnAggregationStatsRule.java new file mode 100644 index 0000000000000000000000000000000000000000..4adc9b692daf97a2a165b32877ca59fd8f3bc7f5 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/cost/JoinOnAggregationStatsRule.java @@ -0,0 +1,253 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.cost; + +import com.google.common.annotations.VisibleForTesting; +import io.prestosql.Session; +import io.prestosql.matching.Pattern; +import io.prestosql.spi.plan.JoinNode.EquiJoinClause; +import io.prestosql.spi.plan.JoinOnAggregationNode; +import io.prestosql.spi.plan.Symbol; +import io.prestosql.sql.planner.TypeProvider; +import io.prestosql.sql.planner.iterative.Lookup; +import io.prestosql.sql.tree.ComparisonExpression; +import io.prestosql.util.MoreMath; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.prestosql.SystemSessionProperties.getJoinMultiClauseIndependenceFactor; +import static io.prestosql.cost.FilterStatsCalculator.UNKNOWN_FILTER_COEFFICIENT; +import static io.prestosql.cost.PlanNodeStatsEstimateMath.estimateCorrelatedConjunctionRowCount; +import static io.prestosql.cost.SymbolStatsEstimate.buildFrom; +import static io.prestosql.sql.planner.SymbolUtils.toSymbolReference; +import static io.prestosql.sql.planner.plan.Patterns.joinOnAggregation; +import static io.prestosql.sql.relational.OriginalExpressionUtils.castToExpression; +import static io.prestosql.sql.relational.OriginalExpressionUtils.isExpression; +import static io.prestosql.sql.tree.ComparisonExpression.Operator.EQUAL; +import static java.lang.Double.isNaN; +import static java.util.Objects.requireNonNull; + +public class JoinOnAggregationStatsRule + extends SimpleStatsRule +{ + private static final Pattern PATTERN = joinOnAggregation(); + private static final double DEFAULT_UNMATCHED_JOIN_COMPLEMENT_NDVS_COEFFICIENT = 0.5; + + private final FilterStatsCalculator filterStatsCalculator; + private final StatsNormalizer normalizer; + private final double unmatchedJoinComplementNdvsCoefficient; + + public JoinOnAggregationStatsRule(FilterStatsCalculator filterStatsCalculator, StatsNormalizer normalizer) + { + this(filterStatsCalculator, normalizer, DEFAULT_UNMATCHED_JOIN_COMPLEMENT_NDVS_COEFFICIENT); + } + + @VisibleForTesting + JoinOnAggregationStatsRule(FilterStatsCalculator filterStatsCalculator, StatsNormalizer normalizer, double unmatchedJoinComplementNdvsCoefficient) + { + super(normalizer); + this.filterStatsCalculator = requireNonNull(filterStatsCalculator, "filterStatsCalculator is null"); + this.normalizer = normalizer; + this.unmatchedJoinComplementNdvsCoefficient = unmatchedJoinComplementNdvsCoefficient; + } + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + protected Optional doCalculate(JoinOnAggregationNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) + { + PlanNodeStatsEstimate leftStats = sourceStats.getStats(node.getLeft()); + PlanNodeStatsEstimate rightStats = sourceStats.getStats(node.getRight()); + PlanNodeStatsEstimate leftAggrStats = AggregationStatsRule.groupBy(leftStats, node.getLeftAggr().getGroupingKeys(), node.getLeftAggr().getAggregations()); + PlanNodeStatsEstimate rightAggrStats = AggregationStatsRule.groupBy(rightStats, node.getRightAggr().getGroupingKeys(), node.getRightAggr().getAggregations()); + + PlanNodeStatsEstimate crossJoinStats = crossJoinStats(node, leftAggrStats, rightAggrStats, types); + + switch (node.getType()) { + case INNER: + // TODO Vineet Check - how to consider cost of AggrOnAggr after join + return Optional.of(computeInnerJoinStats(node, crossJoinStats, session, types)); + default: + throw new IllegalStateException("Unknown group join type: " + node.getType()); + } + } + + private PlanNodeStatsEstimate computeInnerJoinStats(JoinOnAggregationNode node, PlanNodeStatsEstimate crossJoinStats, Session session, TypeProvider types) + { + List equiJoinCriteria = node.getCriteria(); + + Map layout = new HashMap<>(); + int channel = 0; + for (Symbol symbol : node.getOutputSymbols()) { + layout.put(channel++, symbol); + } + + if (equiJoinCriteria.isEmpty()) { + if (!node.getFilter().isPresent()) { + return crossJoinStats; + } + // TODO: this might explode stats + if (isExpression(node.getFilter().get())) { + return filterStatsCalculator.filterStats(crossJoinStats, castToExpression(node.getFilter().get()), session, types); + } + else { + return filterStatsCalculator.filterStats(crossJoinStats, node.getFilter().get(), session, types, layout); + } + } + + PlanNodeStatsEstimate equiJoinEstimate = filterByEquiJoinClauses(crossJoinStats, node.getCriteria(), session, types); + + if (equiJoinEstimate.isOutputRowCountUnknown()) { + return PlanNodeStatsEstimate.unknown(); + } + + if (!node.getFilter().isPresent()) { + return equiJoinEstimate; + } + + PlanNodeStatsEstimate filteredEquiJoinEstimate; + if (isExpression(node.getFilter().get())) { + filteredEquiJoinEstimate = filterStatsCalculator.filterStats(equiJoinEstimate, castToExpression(node.getFilter().get()), session, types); + } + else { + filteredEquiJoinEstimate = filterStatsCalculator.filterStats(equiJoinEstimate, node.getFilter().get(), session, types, layout); + } + + if (filteredEquiJoinEstimate.isOutputRowCountUnknown()) { + return normalizer.normalize(equiJoinEstimate.mapOutputRowCount(rowCount -> rowCount * UNKNOWN_FILTER_COEFFICIENT), types); + } + + return filteredEquiJoinEstimate; + } + + private PlanNodeStatsEstimate filterByEquiJoinClauses( + PlanNodeStatsEstimate stats, + Collection clauses, + Session session, + TypeProvider types) + { + checkArgument(!clauses.isEmpty(), "clauses is empty"); + // Join equality clauses are usually correlated. Therefore, we shouldn't treat each join equality + // clause separately because stats estimates would be way off. + List knownEstimates = clauses.stream() + .map(clause -> { + ComparisonExpression predicate = new ComparisonExpression(EQUAL, toSymbolReference(clause.getLeft()), toSymbolReference(clause.getRight())); + return new PlanNodeStatsEstimateWithClause(filterStatsCalculator.filterStats(stats, predicate, session, types), clause); + }) + .collect(toImmutableList()); + + double outputRowCount = estimateCorrelatedConjunctionRowCount( + stats, + knownEstimates.stream().map(PlanNodeStatsEstimateWithClause::getEstimate).collect(toImmutableList()), + getJoinMultiClauseIndependenceFactor(session)); + if (isNaN(outputRowCount)) { + return PlanNodeStatsEstimate.unknown(); + } + return normalizer.normalize(new PlanNodeStatsEstimate(outputRowCount, intersectCorrelatedJoinClause(stats, knownEstimates)), types); + } + + private static double firstNonNaN(double... values) + { + for (double value : values) { + if (!isNaN(value)) { + return value; + } + } + throw new IllegalArgumentException("All values are NaN"); + } + + private PlanNodeStatsEstimate crossJoinStats(JoinOnAggregationNode node, PlanNodeStatsEstimate leftStats, PlanNodeStatsEstimate rightStats, TypeProvider types) + { + PlanNodeStatsEstimate.Builder builder = PlanNodeStatsEstimate.builder() + .setOutputRowCount(leftStats.getOutputRowCount() * rightStats.getOutputRowCount()); + + node.getLeft().getOutputSymbols().forEach(symbol -> builder.addSymbolStatistics(symbol, leftStats.getSymbolStatistics(symbol))); + node.getRight().getOutputSymbols().forEach(symbol -> builder.addSymbolStatistics(symbol, rightStats.getSymbolStatistics(symbol))); + + return normalizer.normalize(builder.build(), types); + } + + private static Map intersectCorrelatedJoinClause( + PlanNodeStatsEstimate stats, + List equiJoinClauseEstimates) + { + // Add initial statistics (including stats for columns which are not part of equi-join clauses) + PlanNodeStatsEstimate.Builder result = PlanNodeStatsEstimate.builder() + .addSymbolStatistics(stats.getSymbolStatistics()); + + for (PlanNodeStatsEstimateWithClause estimateWithClause : equiJoinClauseEstimates) { + EquiJoinClause clause = estimateWithClause.getClause(); + // we just clear null fraction and adjust ranges here, selectivity is handled outside this function + SymbolStatsEstimate leftStats = stats.getSymbolStatistics(clause.getLeft()); + SymbolStatsEstimate rightStats = stats.getSymbolStatistics(clause.getRight()); + StatisticRange leftRange = StatisticRange.from(leftStats); + StatisticRange rightRange = StatisticRange.from(rightStats); + + StatisticRange intersect = leftRange.intersect(rightRange); + double leftFilterValue = firstNonNaN(leftRange.overlapPercentWith(intersect), 1); + double rightFilterValue = firstNonNaN(rightRange.overlapPercentWith(intersect), 1); + double leftNdvInRange = leftFilterValue * leftRange.getDistinctValuesCount(); + double rightNdvInRange = rightFilterValue * rightRange.getDistinctValuesCount(); + double retainedNdv = MoreMath.min(leftNdvInRange, rightNdvInRange); + + SymbolStatsEstimate newLeftStats = buildFrom(leftStats) + .setNullsFraction(0) + .setStatisticsRange(intersect) + .setDistinctValuesCount(retainedNdv) + .build(); + + SymbolStatsEstimate newRightStats = buildFrom(rightStats) + .setNullsFraction(0) + .setStatisticsRange(intersect) + .setDistinctValuesCount(retainedNdv) + .build(); + + result.addSymbolStatistics(clause.getLeft(), newLeftStats) + .addSymbolStatistics(clause.getRight(), newRightStats); + } + return result.build().getSymbolStatistics(); + } + + private static class PlanNodeStatsEstimateWithClause + { + private final PlanNodeStatsEstimate estimate; + private final EquiJoinClause clause; + + private PlanNodeStatsEstimateWithClause(PlanNodeStatsEstimate estimate, EquiJoinClause clause) + { + this.estimate = requireNonNull(estimate, "estimate is null"); + this.clause = requireNonNull(clause, "clause is null"); + } + + private PlanNodeStatsEstimate getEstimate() + { + return estimate; + } + + private EquiJoinClause getClause() + { + return clause; + } + } +} diff --git a/presto-main/src/main/java/io/prestosql/cost/StatsCalculatorModule.java b/presto-main/src/main/java/io/prestosql/cost/StatsCalculatorModule.java index 7463839fcb17f7dec621176d82c8eddc602bd0d1..fe8f0802810b4a880f626bab66e8454b83632acd 100644 --- a/presto-main/src/main/java/io/prestosql/cost/StatsCalculatorModule.java +++ b/presto-main/src/main/java/io/prestosql/cost/StatsCalculatorModule.java @@ -51,6 +51,7 @@ public class StatsCalculatorModule rules.add(new ProjectStatsRule(scalarStatsCalculator, normalizer)); rules.add(new ExchangeStatsRule(normalizer)); rules.add(new JoinStatsRule(filterStatsCalculator, normalizer)); + rules.add(new JoinOnAggregationStatsRule(filterStatsCalculator, normalizer)); rules.add(new SpatialJoinStatsRule(filterStatsCalculator, normalizer)); rules.add(new AggregationStatsRule(normalizer)); rules.add(new UnionStatsRule(normalizer)); diff --git a/presto-main/src/main/java/io/prestosql/dynamicfilter/DynamicFilterService.java b/presto-main/src/main/java/io/prestosql/dynamicfilter/DynamicFilterService.java index ed09b3269cf255916e0559a83fe66fa63501cd9a..1d8b4d2e8d5f6828089ba09895c31a89f18affff 100644 --- a/presto-main/src/main/java/io/prestosql/dynamicfilter/DynamicFilterService.java +++ b/presto-main/src/main/java/io/prestosql/dynamicfilter/DynamicFilterService.java @@ -34,6 +34,7 @@ import io.prestosql.spi.dynamicfilter.DynamicFilter.DataType; import io.prestosql.spi.dynamicfilter.DynamicFilterFactory; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.Symbol; import io.prestosql.spi.relation.CallExpression; @@ -338,6 +339,13 @@ public class DynamicFilterService registerTasksHelper(node, semiJoinNode.getFilteringSourceJoinSymbol(), Collections.singletonMap(semiJoinNode.getDynamicFilterId().get(), semiJoinNode.getFilteringSourceJoinSymbol()), taskIds, workers, stateMachine); } } + else if (node instanceof JoinOnAggregationNode) { + JoinOnAggregationNode joinNode = (JoinOnAggregationNode) node; + List criterias = joinNode.getCriteria(); + if (!criterias.isEmpty()) { + registerTasksHelper(node, criterias.get(0).getRight(), joinNode.getDynamicFilters(), taskIds, workers, stateMachine); + } + } } private void registerTasksHelper(PlanNode node, Symbol buildSymbol, Map dynamicFiltersMap, Set taskIds, Set workers, StageStateMachine stateMachine) @@ -358,6 +366,9 @@ public class DynamicFilterService else if (node instanceof SemiJoinNode) { filters.put(filterId, extractDynamicFilterRegistryInfo((SemiJoinNode) node, stateMachine.getSession())); } + else if (node instanceof JoinOnAggregationNode) { + filters.put(filterId, extractDynamicFilterRegistryInfo((JoinOnAggregationNode) node, stateMachine.getSession(), filterId)); + } dynamicFiltersToTask.putIfAbsent(filterId + "-" + queryId, new CopyOnWriteArraySet<>()); CopyOnWriteArraySet taskSet = dynamicFiltersToTask.get(filterId + "-" + queryId); taskSet.addAll(taskIds); @@ -511,6 +522,43 @@ public class DynamicFilterService } } + private static DynamicFilterRegistryInfo extractDynamicFilterRegistryInfo(JoinOnAggregationNode node, Session session, String filterId) + { + Symbol symbol = node.getCriteria().isEmpty() ? null : node.getCriteria().get(0).getLeft(); + List filterNodes = findFilterNodeInStage(node); + + if (filterNodes.isEmpty()) { + return new DynamicFilterRegistryInfo(symbol, GLOBAL, session, Optional.empty()); + } + else { + Optional> filterPredicate = Optional.empty(); + if (symbol == null) { + //Symbol is not found in Join Node. It must have been pushed down to filters. + for (FilterNode filter : filterNodes) { + DynamicFilters.ExtractResult extractResult = DynamicFilters.extractDynamicFilters(filter.getPredicate()); + List dynamicConjuncts = extractResult.getDynamicConjuncts(); + for (DynamicFilters.Descriptor desc : dynamicConjuncts) { + if (desc.getId().equals(filterId)) { + checkArgument(desc.getInput() instanceof VariableReferenceExpression, "Expression not symbol reference"); + symbol = new Symbol(((VariableReferenceExpression) desc.getInput()).getName()); + if (desc.getFilter().isPresent()) { + filterPredicate = DynamicFilters.createDynamicFilterPredicate(desc.getFilter()); + } + break; + } + } + if (symbol != null) { + break; + } + } + if (symbol == null) { + throw new IllegalStateException("DynamicFilter symbol not found to register"); + } + } + return new DynamicFilterRegistryInfo(symbol, LOCAL, session, filterPredicate); + } + } + private static DynamicFilterRegistryInfo extractDynamicFilterRegistryInfo(SemiJoinNode node, Session session) { Symbol symbol = node.getFilteringSourceJoinSymbol(); @@ -625,6 +673,9 @@ public class DynamicFilterService if (planNode instanceof SemiJoinNode) { return ((SemiJoinNode) planNode).getDynamicFilterId().map(ImmutableSet::of).orElse(ImmutableSet.of()); } + else if (planNode instanceof JoinOnAggregationNode) { + return ((JoinOnAggregationNode) planNode).getDynamicFilters().keySet(); + } throw new IllegalStateException("getDynamicFiltersProducedInPlanNode called with neither JoinNode nor SemiJoinNode"); } diff --git a/presto-main/src/main/java/io/prestosql/execution/SqlStageExecution.java b/presto-main/src/main/java/io/prestosql/execution/SqlStageExecution.java index ad847703ee6b4e4e6123d942fc50dc63b7f1060e..e4c3cf90ef2bf9a29b0ed5ba5333e7060cde5f6c 100644 --- a/presto-main/src/main/java/io/prestosql/execution/SqlStageExecution.java +++ b/presto-main/src/main/java/io/prestosql/execution/SqlStageExecution.java @@ -43,6 +43,7 @@ import io.prestosql.snapshot.QuerySnapshotManager; import io.prestosql.spi.PrestoException; import io.prestosql.spi.QueryId; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeId; import io.prestosql.spi.plan.TableScanNode; @@ -247,6 +248,10 @@ public final class SqlStageExecution SemiJoinNode semiJoinNode = (SemiJoinNode) node; dynamicFilterService.registerTasks(semiJoinNode, allTasks, getScheduledNodes(), stateMachine); } + else if (node instanceof JoinOnAggregationNode) { + JoinOnAggregationNode joinNode = (JoinOnAggregationNode) node; + dynamicFilterService.registerTasks(joinNode, allTasks, getScheduledNodes(), stateMachine); + } traverseNodesForDynamicFiltering(node.getSources()); } } diff --git a/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/AllAtOnceExecutionSchedule.java b/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/AllAtOnceExecutionSchedule.java index c9e504ad58105acedee8f4e4bcf0a13902f93dda..1aea5fa7cc6725238cabf425046d5a53d04aef9d 100644 --- a/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/AllAtOnceExecutionSchedule.java +++ b/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/AllAtOnceExecutionSchedule.java @@ -20,6 +20,7 @@ import com.google.common.collect.Ordering; import io.prestosql.execution.SqlStageExecution; import io.prestosql.execution.StageState; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.UnionNode; import io.prestosql.sql.planner.PlanFragment; @@ -140,6 +141,14 @@ public class AllAtOnceExecutionSchedule return null; } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + node.getRight().accept(this, context); + node.getLeft().accept(this, context); + return null; + } + @Override public Void visitSemiJoin(SemiJoinNode node, Void context) { diff --git a/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/PhasedExecutionSchedule.java b/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/PhasedExecutionSchedule.java index 914d44402ead727f206ca0c7a64682de9ed14cd3..a9f16faff26c00b9a92d2fda55caa48f6bb1a9d8 100644 --- a/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/PhasedExecutionSchedule.java +++ b/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/PhasedExecutionSchedule.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableSet; import io.prestosql.execution.SqlStageExecution; import io.prestosql.execution.StageState; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.UnionNode; import io.prestosql.sql.planner.PlanFragment; @@ -205,6 +206,12 @@ public class PhasedExecutionSchedule return processJoin(node.getRight(), node.getLeft(), currentFragmentId); } + @Override + public Set visitJoinOnAggregation(JoinOnAggregationNode node, PlanFragmentId currentFragmentId) + { + return processJoin(node.getRight(), node.getLeft(), currentFragmentId); + } + @Override public Set visitSpatialJoin(SpatialJoinNode node, PlanFragmentId currentFragmentId) { diff --git a/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/PrioritizeUtilizationExecutionSchedule.java b/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/PrioritizeUtilizationExecutionSchedule.java index 2b3cf878367d3237844b16d28847ad97f37e0fc3..fce5c90eb03e35f831fd3ed37bccff9e745dc598 100644 --- a/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/PrioritizeUtilizationExecutionSchedule.java +++ b/presto-main/src/main/java/io/prestosql/execution/scheduler/policy/PrioritizeUtilizationExecutionSchedule.java @@ -27,6 +27,7 @@ import io.prestosql.spi.QueryId; import io.prestosql.spi.plan.AggregationNode; import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.sql.planner.PlanFragment; import io.prestosql.sql.planner.plan.CacheTableWriterNode; @@ -367,6 +368,16 @@ public class PrioritizeUtilizationExecutionSchedule currentFragmentId); } + @Override + public FragmentSubGraph visitJoinOnAggregation(JoinOnAggregationNode node, PlanFragmentId currentFragmentId) + { + return processJoin( + node.getDistributionType().orElseThrow(() -> new NoSuchElementException("No Value Present")) == JoinNode.DistributionType.REPLICATED, + node.getLeft(), + node.getRight(), + currentFragmentId); + } + @Override public FragmentSubGraph visitSpatialJoin(SpatialJoinNode node, PlanFragmentId currentFragmentId) { diff --git a/presto-main/src/main/java/io/prestosql/operator/BigintPagesHash.java b/presto-main/src/main/java/io/prestosql/operator/BigintPagesHash.java index 70a04983a2233bd7cf037f6351723acd8270aa1f..77751bee6cf8810df279c86660977cf4fe041ee7 100644 --- a/presto-main/src/main/java/io/prestosql/operator/BigintPagesHash.java +++ b/presto-main/src/main/java/io/prestosql/operator/BigintPagesHash.java @@ -15,6 +15,7 @@ package io.prestosql.operator; import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; import io.prestosql.spi.block.Block; @@ -258,6 +259,21 @@ public final class BigintPagesHash return result; } + @Override + public long getCountForJoinPosition(long position, int channel) + { + long pageAddress = addresses.getLong(toIntExact(position)); + int blockIndex = decodeSliceIndex(pageAddress); + int blockPosition = decodePosition(pageAddress); + return pagesHashStrategy.getCountForJoinPosition(blockIndex, blockPosition, channel); + } + + @Override + public AggregationBuilder getAggregationBuilder() + { + return pagesHashStrategy.getAggregationBuilder(); + } + @Override public void appendTo(long position, PageBuilder pageBuilder, int outputChannelOffset) { diff --git a/presto-main/src/main/java/io/prestosql/operator/DefaultPagesHash.java b/presto-main/src/main/java/io/prestosql/operator/DefaultPagesHash.java index cc97d06ce01d0071294802d08028473828f46c99..ec34b2df964392e5a21d6ac9905e74232ea1bc08 100644 --- a/presto-main/src/main/java/io/prestosql/operator/DefaultPagesHash.java +++ b/presto-main/src/main/java/io/prestosql/operator/DefaultPagesHash.java @@ -14,6 +14,7 @@ package io.prestosql.operator; import io.airlift.units.DataSize; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; import it.unimi.dsi.fastutil.HashCommon; @@ -170,6 +171,21 @@ public final class DefaultPagesHash return -1; } + @Override + public long getCountForJoinPosition(long position, int channel) + { + long pageAddress = addresses.getLong(toIntExact(position)); + int blockIndex = decodeSliceIndex(pageAddress); + int blockPosition = decodePosition(pageAddress); + return pagesHashStrategy.getCountForJoinPosition(blockIndex, blockPosition, channel); + } + + @Override + public AggregationBuilder getAggregationBuilder() + { + return pagesHashStrategy.getAggregationBuilder(); + } + public void appendTo(long position, PageBuilder pageBuilder, int outputChannelOffset) { long pageAddress = addresses.getLong(toIntExact(position)); diff --git a/presto-main/src/main/java/io/prestosql/operator/GroupJoinAggregator.java b/presto-main/src/main/java/io/prestosql/operator/GroupJoinAggregator.java new file mode 100644 index 0000000000000000000000000000000000000000..0ec4ba17763f34425758486588c077bd8a165300 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/GroupJoinAggregator.java @@ -0,0 +1,257 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator; + +import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; +import io.prestosql.operator.aggregation.Accumulator; +import io.prestosql.operator.aggregation.AccumulatorFactory; +import io.prestosql.operator.aggregation.partial.PartialAggregationController; +import io.prestosql.operator.scalar.CombineHashFunction; +import io.prestosql.spi.Page; +import io.prestosql.spi.PageBuilder; +import io.prestosql.spi.plan.AggregationNode; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.Type; +import io.prestosql.sql.gen.JoinCompiler; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.prestosql.operator.aggregation.builder.InMemoryHashAggregationBuilder.toTypes; +import static io.prestosql.sql.planner.optimizations.HashGenerationOptimizer.INITIAL_HASH_VALUE; +import static io.prestosql.type.TypeUtils.NULL_HASH_CODE; +import static java.util.Objects.requireNonNull; + +public class GroupJoinAggregator +{ + private final Optional hashChannel; + private final Optional groupIdChannel; + private final List accumulatorFactories; + private final List groupByTypes; + private final List groupByChannels; + private final List globalAggregationGroupIds; + private final AggregationNode.Step step; + private final int expectedGroups; + private final Optional maxPartialMemory; + private final JoinCompiler joinCompiler; + private final boolean useSystemMemory; + private final Optional partialAggregationController; + private final boolean produceDefaultOutput; + + protected final List types; + + public GroupJoinAggregator(Optional hashChannel, + Optional groupIdChannel, + List accumulatorFactories, + List groupByTypes, + List groupByChannels, + List globalAggregationGroupIds, + AggregationNode.Step step, + int expectedGroups, + Optional maxPartialMemory, + JoinCompiler joinCompiler, + boolean useSystemMemory, + Optional partialAggregationController, + boolean produceDefaultOutput) + { + requireNonNull(accumulatorFactories, "accumulatorFactories is null"); + checkArgument(!partialAggregationController.isPresent() || step.isOutputPartial(), + "partialAggregationController should be present only for partial aggregation"); + this.hashChannel = requireNonNull(hashChannel, "hashChannel is null"); + this.groupIdChannel = requireNonNull(groupIdChannel, "groupIdChannel is null"); + this.accumulatorFactories = ImmutableList.copyOf(accumulatorFactories); + this.groupByTypes = ImmutableList.copyOf(groupByTypes); + this.groupByChannels = ImmutableList.copyOf(groupByChannels); + this.globalAggregationGroupIds = ImmutableList.copyOf(globalAggregationGroupIds); + this.step = requireNonNull(step, "step is null"); + this.expectedGroups = expectedGroups; + this.maxPartialMemory = requireNonNull(maxPartialMemory, "maxPartialMemory is null"); + this.joinCompiler = requireNonNull(joinCompiler, "joinCompiler is null"); + this.useSystemMemory = useSystemMemory; + this.partialAggregationController = requireNonNull(partialAggregationController, "partialAggregationController is null"); + this.produceDefaultOutput = produceDefaultOutput; + this.types = toTypes(groupByTypes, step, accumulatorFactories, hashChannel); + } + + public Optional getHashChannel() + { + return hashChannel; + } + + public Optional getGroupIdChannel() + { + return groupIdChannel; + } + + public List getAccumulatorFactories() + { + return accumulatorFactories; + } + + public List getGroupByTypes() + { + return groupByTypes; + } + + public List getGroupByChannels() + { + return groupByChannels; + } + + public List getGlobalAggregationGroupIds() + { + return globalAggregationGroupIds; + } + + public AggregationNode.Step getStep() + { + return step; + } + + public int getExpectedGroups() + { + return expectedGroups; + } + + public Optional getMaxPartialMemory() + { + return maxPartialMemory; + } + + public JoinCompiler getJoinCompiler() + { + return joinCompiler; + } + + public boolean isUseSystemMemory() + { + return useSystemMemory; + } + + public Optional getPartialAggregationController() + { + return partialAggregationController; + } + + public boolean isProduceDefaultOutput() + { + return produceDefaultOutput; + } + + public List getTypes() + { + return types; + } + + protected Page getGlobalAggregationOutput() + { + List accumulators = accumulatorFactories.stream() + .map(AccumulatorFactory::createAccumulator) + .collect(Collectors.toList()); + + // global aggregation output page will only be constructed once, + // so a new PageBuilder is constructed (instead of using PageBuilder.reset) + PageBuilder output = new PageBuilder(globalAggregationGroupIds.size(), types); + + for (int groupId : globalAggregationGroupIds) { + output.declarePosition(); + int channel = 0; + + for (; channel < groupByTypes.size(); channel++) { + if (channel == groupIdChannel.get()) { + output.getBlockBuilder(channel).writeLong(groupId); + } + else { + output.getBlockBuilder(channel).appendNull(); + } + } + + if (hashChannel.isPresent()) { + long hashValue = calculateDefaultOutputHash(groupByTypes, groupIdChannel.get(), groupId); + output.getBlockBuilder(channel++).writeLong(hashValue); + } + + for (int j = 0; j < accumulators.size(); channel++, j++) { + if (step.isOutputPartial()) { + accumulators.get(j).evaluateIntermediate(output.getBlockBuilder(channel)); + } + else { + accumulators.get(j).evaluateFinal(output.getBlockBuilder(channel)); + } + } + } + + if (output.isEmpty()) { + return null; + } + return output.build(); + } + + private static long calculateDefaultOutputHash(List groupByChannels, int groupIdChannel, int groupId) + { + // Default output has NULLs on all columns except of groupIdChannel + long result = INITIAL_HASH_VALUE; + for (int channel = 0; channel < groupByChannels.size(); channel++) { + if (channel != groupIdChannel) { + result = CombineHashFunction.getHash(result, NULL_HASH_CODE); + } + else { + result = CombineHashFunction.getHash(result, BigintType.hash(groupId)); + } + } + return result; + } + + protected boolean hasOrderBy() + { + return accumulatorFactories.stream().anyMatch(AccumulatorFactory::hasOrderBy); + } + + protected boolean hasDistinct() + { + return accumulatorFactories.stream().anyMatch(AccumulatorFactory::hasDistinct); + } + + public static GroupJoinAggregator buildGroupJoinAggregator(List groupByTypes, + List groupByChannels, + List globalAggregationGroupIds, + AggregationNode.Step step, + List accumulatorFactories, + Optional hashChannel, + Optional groupIdChannel, + int expectedGroups, + Optional maxPartialMemory, + JoinCompiler joinCompiler, + boolean useSystemMemory, + Optional partialAggregationController, + boolean produceDefaultOutput) + { + return new GroupJoinAggregator(hashChannel, + groupIdChannel, + accumulatorFactories, + groupByTypes, + groupByChannels, + globalAggregationGroupIds, + step, + expectedGroups, + maxPartialMemory, + joinCompiler, + useSystemMemory, + partialAggregationController, + produceDefaultOutput); + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/GroupJoinProbe.java b/presto-main/src/main/java/io/prestosql/operator/GroupJoinProbe.java new file mode 100644 index 0000000000000000000000000000000000000000..e0b834348e2df75291d898030bfb23ef4324e1e6 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/GroupJoinProbe.java @@ -0,0 +1,145 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator; + +import io.prestosql.operator.aggregation.builder.AggregationBuilder; +import io.prestosql.spi.Page; +import io.prestosql.spi.block.Block; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; + +import static io.prestosql.spi.type.BigintType.BIGINT; +import static java.util.Objects.requireNonNull; + +public class GroupJoinProbe +{ + public static class GroupJoinProbeFactory + { + private final int[] probeOutputChannels; + private final List probeJoinChannels; + private final OptionalInt probeHashChannel; + private final OptionalInt probeCountChannel; + + public GroupJoinProbeFactory(int[] probeOutputChannels, List probeJoinChannels, OptionalInt probeHashChannel, OptionalInt probeCountChannel) + { + this.probeOutputChannels = probeOutputChannels; + this.probeJoinChannels = probeJoinChannels; + this.probeHashChannel = probeHashChannel; + this.probeCountChannel = probeCountChannel; + } + + public GroupJoinProbe createGroupJoinProbe(Page page, boolean isSpilled, LookupSourceProvider lookupSourceProvider, + AggregationBuilder probeAggregationBuilder, AggregationBuilder buildAggregationBuilder) + { + LookupSource lookupSource = lookupSourceProvider.withLease((lookupSourceLease -> lookupSourceLease.getLookupSource())); + if (isSpilled || !(lookupSource instanceof JoinHash || lookupSource instanceof OuterLookupSource)) { + return new GroupJoinProbe(probeOutputChannels, page, probeJoinChannels, probeHashChannel, probeCountChannel, probeAggregationBuilder, buildAggregationBuilder); + } + else { + Page loadedProbePage = page.getLoadedPage(probeJoinChannels.stream().mapToInt(i -> i).toArray()); + return new UnSpilledGroupJoinProbe(probeOutputChannels, page, probeJoinChannels, probeHashChannel, loadedProbePage, lookupSource, (probeHashChannel.isPresent() ? (probeHashChannel.getAsInt() >= 0 ? page.getBlock(probeHashChannel.getAsInt()).getLoadedBlock() : null) : null), probeCountChannel, probeAggregationBuilder, buildAggregationBuilder); + } + } + } + + private final int[] probeOutputChannels; + private final int positionCount; + private final Block[] probeBlocks; + private final AggregationBuilder probeAggregationBuilder; + private final AggregationBuilder buildAggregationBuilder; + private final Page page; + private final Page probePage; + private final Optional probeHashBlock; + private final Optional probeCountBlock; + + private int position = -1; + + protected GroupJoinProbe(int[] probeOutputChannels, Page page, List probeJoinChannels, OptionalInt probeHashChannel, + OptionalInt probeCountChannel, AggregationBuilder probeAggregationBuilder, AggregationBuilder buildAggregationBuilder) + { + this.probeOutputChannels = probeOutputChannels; + this.positionCount = page.getPositionCount(); + this.probeBlocks = new Block[probeJoinChannels.size()]; + this.probeAggregationBuilder = requireNonNull(probeAggregationBuilder, "probeAggregationBuilder is null"); + this.buildAggregationBuilder = requireNonNull(buildAggregationBuilder, "buildAggregationBuilder is null"); + + for (int i = 0; i < probeJoinChannels.size(); i++) { + probeBlocks[i] = page.getBlock(probeJoinChannels.get(i)); + } + this.page = page; + this.probePage = new Page(page.getPositionCount(), probeBlocks); + this.probeHashBlock = probeHashChannel.isPresent() ? Optional.of(page.getBlock(probeHashChannel.getAsInt())) : Optional.empty(); + this.probeCountBlock = probeCountChannel.isPresent() ? Optional.of(page.getBlock(probeCountChannel.getAsInt())) : Optional.empty(); + } + + public AggregationBuilder getProbeAggregationBuilder() + { + return probeAggregationBuilder; + } + + public AggregationBuilder getBuildAggregationBuilder() + { + return buildAggregationBuilder; + } + + public int[] getOutputChannels() + { + return probeOutputChannels; + } + + public boolean advanceNextPosition() + { + position++; + return position < positionCount; + } + + public long getCurrentJoinPosition(LookupSource lookupSource) + { + if (currentRowContainsNull()) { + return -1; + } + if (probeHashBlock.isPresent()) { + long rawHash = BIGINT.getLong(probeHashBlock.get(), position); + return lookupSource.getJoinPosition(position, probePage, page, rawHash); + } + return lookupSource.getJoinPosition(position, probePage, page); + } + + public int getPosition() + { + return position; + } + + public Page getPage() + { + return page; + } + + private boolean currentRowContainsNull() + { + for (Block probeBlock : probeBlocks) { + if (probeBlock.isNull(position)) { + return true; + } + } + return false; + } + + public long getCountProbeRecord() + { + return BIGINT.getLong(probeCountBlock.get(), position); + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/HashBuilderGroupJoinOperator.java b/presto-main/src/main/java/io/prestosql/operator/HashBuilderGroupJoinOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..3472a4b5c58b5e3bf38d6fdda826aae743c8df9f --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/HashBuilderGroupJoinOperator.java @@ -0,0 +1,584 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.io.Closer; +import com.google.common.util.concurrent.ListenableFuture; +import io.prestosql.memory.context.LocalMemoryContext; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; +import io.prestosql.operator.aggregation.builder.InMemoryHashAggregationBuilder; +import io.prestosql.operator.aggregation.builder.InMemoryHashAggregationBuilderWithReset; +import io.prestosql.operator.groupjoin.ExecutionHelperFactory; +import io.prestosql.spi.Page; +import io.prestosql.sql.gen.JoinFilterFunctionCompiler; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Verify.verify; +import static java.util.Objects.requireNonNull; + +public class HashBuilderGroupJoinOperator + implements SinkOperator +{ + private ExecutionHelperFactory executionHelperFactory; + + @VisibleForTesting + public enum State + { + /** + * Operator accepts input + */ + CONSUMING_INPUT, + + /** + * All inputs accepted, finishing aggregations + */ + AGGR_FINISHING, + + /** + * Aggregation on input finished + */ + AGGR_FINISHED, + + /** + * LookupSource has been built and passed on without any spill occurring + */ + LOOKUP_SOURCE_BUILT, + + /** + * No longer needed + */ + CLOSED + } + + private final OperatorContext operatorContext; + private final LocalMemoryContext localUserMemoryContext; + private final LocalMemoryContext localRevocableMemoryContext; + //TODO-cp-I2DSGR: Shared field + private final PartitionedLookupSourceFactory lookupSourceFactory; + private final ListenableFuture lookupSourceFactoryDestroyed; + private final int partitionIndex; + private final List outputChannels; + private final List hashChannels; + private final OptionalInt preComputedHashChannel; + private final Optional filterFunctionFactory; + private final Optional sortChannel; + private final Optional countChannel; + private final List searchFunctionFactories; + private final PagesIndex index; + private final boolean spillEnabled; + private final HashCollisionsCounter hashCollisionsCounter; + private State state = State.CONSUMING_INPUT; + private Optional> lookupSourceNotNeeded = Optional.empty(); + @Nullable + private LookupSourceSupplier lookupSourceSupplier; + private Optional finishMemoryRevoke = Optional.empty(); + private boolean spillToHdfsEnabled; + + protected AggregationBuilder aggregationBuilder; + protected AggregationBuilder aggrOnAggregationBuilder; + protected LocalMemoryContext aggrMemoryContext; + protected LocalMemoryContext aggrOnAggrMemoryContext; + protected WorkProcessor aggrOutputPages; + + // for yield when memory is not available + protected Work unfinishedAggrWork; + protected long numberOfInputRowsProcessed; + protected long numberOfUniqueRowsProduced; + private final GroupJoinAggregator aggregator; + private final GroupJoinAggregator aggrOnAggregator; + private boolean aggregationInputProcessed; + private ListenableFuture executionHelper = NOT_BLOCKED; + + public HashBuilderGroupJoinOperator( + OperatorContext operatorContext, + PartitionedLookupSourceFactory lookupSourceFactory, + int partitionIndex, + List outputChannels, + List hashChannels, + OptionalInt preComputedHashChannel, + Optional filterFunctionFactory, + Optional sortChannel, + Optional countChannel, + List searchFunctionFactories, + int expectedPositions, + PagesIndex.Factory pagesIndexFactory, + boolean spillEnabled, + boolean spillToHdfsEnabled, + GroupJoinAggregator aggregator, + GroupJoinAggregator aggrOnAggregator, + ExecutionHelperFactory executionHelperFactory) + { + this.executionHelperFactory = executionHelperFactory; + requireNonNull(pagesIndexFactory, "pagesIndexFactory is null"); + this.operatorContext = operatorContext; + this.partitionIndex = partitionIndex; + this.filterFunctionFactory = filterFunctionFactory; + this.sortChannel = sortChannel; + this.countChannel = countChannel; + this.searchFunctionFactories = searchFunctionFactories; + this.localUserMemoryContext = operatorContext.localUserMemoryContext(); + this.localRevocableMemoryContext = operatorContext.localRevocableMemoryContext(); + + this.index = pagesIndexFactory.newPagesIndex(lookupSourceFactory.getTypes(), expectedPositions); + this.lookupSourceFactory = lookupSourceFactory; + this.lookupSourceFactoryDestroyed = lookupSourceFactory.isDestroyed(); + + this.outputChannels = outputChannels; + this.hashChannels = hashChannels; + this.preComputedHashChannel = preComputedHashChannel; + + this.hashCollisionsCounter = new HashCollisionsCounter(operatorContext); + operatorContext.setInfoSupplier(hashCollisionsCounter); + // TODO Vineet Use the actual spillEnabled flag when spill is supported + this.spillEnabled = false; + this.spillToHdfsEnabled = false; + + requireNonNull(operatorContext, "operatorContext is null"); + this.aggrMemoryContext = operatorContext.localUserMemoryContext(); + this.aggrOnAggrMemoryContext = operatorContext.localUserMemoryContext(); + if (aggregator.isUseSystemMemory()) { + this.aggrMemoryContext = operatorContext.localSystemMemoryContext(); + } + if (aggrOnAggregator.isUseSystemMemory()) { + this.aggrOnAggrMemoryContext = operatorContext.localSystemMemoryContext(); + } + + this.aggregator = aggregator; + this.aggrOnAggregator = aggrOnAggregator; + createAggrOnAggregationBuilder(); + } + + @Override + public OperatorContext getOperatorContext() + { + return operatorContext; + } + + @Override + public boolean needsInput() + { + if (state == State.CONSUMING_INPUT) { + if (aggrOutputPages != null) { + if (executionHelper != null) { + checkAndResetExeHelper(); + return false; + } + executionHelper = executionHelperFactory.create().submitWork(() -> { + processAggrOutputOrFullCase(); + executionHelper = null; + }); + return false; + } + else if (aggregationBuilder != null && aggregationBuilder.isFull()) { + if (executionHelper != null) { + checkAndResetExeHelper(); + return false; + } + executionHelper = executionHelperFactory.create().submitWork(() -> { + processAggrOutputOrFullCase(); + executionHelper = null; + }); + return false; + } + else if (lookupSourceFactoryDestroyed.isDone()) { + return false; + } + else { + // TODO Vineet Need to move this out of needsInput and need to make it light weight. + if (unfinishedAggrWork != null) { + if (executionHelper != null) { + checkAndResetExeHelper(); + return false; + } + executionHelper = executionHelperFactory.create().submitWork(() -> { + boolean workDone = unfinishedAggrWork.process(); + aggregationBuilder.updateMemory(); + while (!workDone) { + workDone = unfinishedAggrWork.process(); + aggregationBuilder.updateMemory(); + } + + unfinishedAggrWork = null; + executionHelper = null; + }); + return false; + } + return true; + } + } + return false; + } + + private void checkAndResetExeHelper() + { + if (executionHelper.isDone()) { + executionHelper = null; + } + } + + @Override + public void addInput(Page page) + { + requireNonNull(page, "page is null"); + checkState(unfinishedAggrWork == null, "Operator has unfinished work"); + checkState(state == State.CONSUMING_INPUT, "Operator is not in Consuming Input state"); + + if (aggregationBuilder == null) { + createAggregationBuilder(); + } + else { + checkState(!aggregationBuilder.isFull(), "Aggregation buffer is full"); + } + aggregationInputProcessed = true; + + // process the current page; save the unfinished work if we are waiting for memory + unfinishedAggrWork = aggregationBuilder.processPage(page); + if (unfinishedAggrWork.process()) { + unfinishedAggrWork = null; + } + aggregationBuilder.updateMemory(); + numberOfInputRowsProcessed += page.getPositionCount(); + } + + protected boolean hasOrderBy() + { + return aggregator.hasOrderBy(); + } + + protected boolean hasDistinct() + { + return aggregator.hasDistinct(); + } + + public void createAggrOnAggregationBuilder() + { + if (aggrOnAggregator.getStep().isOutputPartial() || !spillEnabled || hasOrderBy() || hasDistinct()) { + aggrOnAggregationBuilder = new InMemoryHashAggregationBuilderWithReset( + aggrOnAggregator.getAccumulatorFactories(), + aggrOnAggregator.getStep(), + aggrOnAggregator.getExpectedGroups(), + aggrOnAggregator.getGroupByTypes(), + aggrOnAggregator.getGroupByChannels(), + aggrOnAggregator.getHashChannel(), + operatorContext, + aggrOnAggregator.getMaxPartialMemory(), + aggrOnAggregator.getJoinCompiler(), + () -> { + aggrOnAggrMemoryContext.setBytes(((InMemoryHashAggregationBuilder) aggrOnAggregationBuilder).getSizeInMemory()); + if (aggrOnAggregator.getStep().isOutputPartial() && aggrOnAggregator.getMaxPartialMemory().isPresent()) { + // do not yield on memory for partial aggregations + return true; + } + return operatorContext.isWaitingForMemory().isDone(); + }); + } + else { + throw new UnsupportedOperationException("Not Supported"); + } + } + + public void createAggregationBuilder() + { + if (aggregator.getStep().isOutputPartial() || !spillEnabled || hasOrderBy() || hasDistinct()) { + // TODO: We ignore spillEnabled here if any aggregate has ORDER BY clause or DISTINCT because they are not yet implemented for spilling. + aggregationBuilder = new InMemoryHashAggregationBuilder( + aggregator.getAccumulatorFactories(), + aggregator.getStep(), + aggregator.getExpectedGroups(), + aggregator.getGroupByTypes(), + aggregator.getGroupByChannels(), + aggregator.getHashChannel(), + operatorContext, + aggregator.getMaxPartialMemory(), + aggregator.getJoinCompiler(), + () -> { + aggrMemoryContext.setBytes(((InMemoryHashAggregationBuilder) aggregationBuilder).getSizeInMemory()); + if (aggregator.getStep().isOutputPartial() && aggregator.getMaxPartialMemory().isPresent()) { + // do not yield on memory for partial aggregations + return true; + } + return operatorContext.isWaitingForMemory().isDone(); + }); + } + else { + throw new UnsupportedOperationException("Not Supported"); + } + } + + private void updateIndex(Page page) + { + index.addPage(page); + if (spillEnabled) { + localRevocableMemoryContext.setBytes(index.getEstimatedSize().toBytes()); + } + else { + if (!localUserMemoryContext.trySetBytes(index.getEstimatedSize().toBytes())) { + index.compact(); + localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes()); + } + } + operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount()); + } + + private void processAggrOutputOrFullCase() + { + Page page = processAggregation(); + if (page != null) { + updateIndex(page); + } + } + + public Page processAggregation() + { + if (state == State.AGGR_FINISHED) { + return null; + } + + // process unfinished work if one exists + if (unfinishedAggrWork != null) { + boolean workDone = unfinishedAggrWork.process(); + aggregationBuilder.updateMemory(); + if (!workDone) { + return null; + } + unfinishedAggrWork = null; + } + + if (aggrOutputPages == null) { + if (!aggregationInputProcessed && aggregator.isProduceDefaultOutput()) { + // global aggregations always generate an output row with the default aggregation output (e.g. 0 for COUNT, NULL for SUM) + state = State.AGGR_FINISHED; + return aggregator.getGlobalAggregationOutput(); + } + + if (aggregationBuilder == null) { + state = State.AGGR_FINISHED; + return null; + } + + // Only flush if we are finishing(consuming input will change to AggrFinishing) or the aggregation builder is full + if (state == State.CONSUMING_INPUT && !aggregationBuilder.isFull()) { + return null; + } + + aggrOutputPages = aggregationBuilder.buildResult(); + } + + if (!aggrOutputPages.process()) { + return null; + } + + if (aggrOutputPages.isFinished()) { + closeAggregationBuilder(); + return null; + } + + Page result = aggrOutputPages.getResult(); + numberOfUniqueRowsProduced += result.getPositionCount(); + return result; + } + + protected void closeAggregationBuilder() + { + aggrOutputPages = null; + if (aggregationBuilder != null) { + aggregationBuilder.recordHashCollisions(hashCollisionsCounter); + aggregationBuilder.close(); + // aggregationBuilder.close() will release all memory reserved in memory accounting. + // The reference must be set to null afterwards to avoid unaccounted memory. + aggregationBuilder = null; + } + aggrMemoryContext.setBytes(0); + aggregator.getPartialAggregationController().ifPresent( + controller -> controller.onFlush(numberOfInputRowsProcessed, numberOfUniqueRowsProduced)); + numberOfInputRowsProcessed = 0; + numberOfUniqueRowsProduced = 0; + } + + protected void closeAggrOnAggregationBuilder() + { + if (aggrOnAggregationBuilder != null) { + aggrOnAggregationBuilder.recordHashCollisions(hashCollisionsCounter); + aggrOnAggregationBuilder.close(); + // aggrOnAggregationBuilder.close() will release all memory reserved in memory accounting. + // The reference must be set to null afterwards to avoid unaccounted memory. + aggrOnAggregationBuilder = null; + } + aggrOnAggrMemoryContext.setBytes(0); + } + + @Override + public void finish() + { + if (state == State.CONSUMING_INPUT) { + state = State.AGGR_FINISHING; + } + doFinish(); + } + + private void doFinish() + { + if (lookupSourceFactoryDestroyed.isDone()) { + close(); + return; + } + + if (finishMemoryRevoke.isPresent()) { + return; + } + + switch (state) { + case CONSUMING_INPUT: + return; + case AGGR_FINISHING: + // finish all aggregation operation first + finishAggregation(); + return; + case AGGR_FINISHED: + finishInput(); + return; + case LOOKUP_SOURCE_BUILT: + disposeLookupSourceIfRequested(); + return; + + case CLOSED: + // no-op + return; + } + + throw new IllegalStateException("Unhandled state: " + state); + } + + private void finishAggregation() + { + DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); + Page page = processAggregation(); + while (page != null) { + updateIndex(page); + if (yieldSignal.isSet()) { + break; + } + page = processAggregation(); + } + } + + private void finishInput() + { + checkState(state == State.AGGR_FINISHED); + if (lookupSourceFactoryDestroyed.isDone()) { + close(); + return; + } + + LookupSourceSupplier partition = buildLookupSource(); + if (spillEnabled) { + localRevocableMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); + } + else { + localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); + } + lookupSourceNotNeeded = Optional.of(lookupSourceFactory.lendPartitionLookupSource(partitionIndex, partition)); + state = State.LOOKUP_SOURCE_BUILT; + } + + private void disposeLookupSourceIfRequested() + { + checkState(state == State.LOOKUP_SOURCE_BUILT); + verify(lookupSourceNotNeeded.isPresent()); + if (!lookupSourceNotNeeded.get().isDone()) { + return; + } + + index.clear(); + localRevocableMemoryContext.setBytes(0); + localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes()); + lookupSourceSupplier = null; + close(); + } + + private LookupSourceSupplier buildLookupSource() + { + LookupSourceSupplier partition = index.createLookupSourceSupplier(operatorContext.getSession(), hashChannels, preComputedHashChannel, + filterFunctionFactory, sortChannel, searchFunctionFactories, Optional.of(outputChannels), + countChannel, Optional.ofNullable(aggrOnAggregationBuilder)); + hashCollisionsCounter.recordHashCollision(partition.getHashCollisions(), partition.getExpectedHashCollisions()); + checkState(lookupSourceSupplier == null, "lookupSourceSupplier is already set"); + this.lookupSourceSupplier = partition; + return partition; + } + + @Override + public boolean isFinished() + { + if (lookupSourceFactoryDestroyed.isDone()) { + // Finish early when the probe side is empty + close(); + return true; + } + + return state == State.CLOSED; + } + + @Override + public void close() + { + if (state == State.CONSUMING_INPUT) { + closeAggregationBuilder(); + } + if (state == State.CLOSED) { + return; + } + // close() can be called in any state, due for example to query failure, and must clean resource up unconditionally + + lookupSourceSupplier = null; + state = State.CLOSED; + finishMemoryRevoke = finishMemoryRevoke.map(ifPresent -> () -> {}); + + try (Closer closer = Closer.create()) { + closer.register(index::clear); + closer.register(() -> localUserMemoryContext.setBytes(0)); + closer.register(() -> localRevocableMemoryContext.setBytes(0)); + closer.register(this::closeAggrOnAggregationBuilder); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ListenableFuture isBlocked() + { + if (state == State.CONSUMING_INPUT) { + if (executionHelper != null) { + return executionHelper.isDone() ? NOT_BLOCKED : executionHelper; + } + } + + return NOT_BLOCKED; + } + + @VisibleForTesting + public State getState() + { + return state; + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/HashBuilderGroupJoinOperatorFactory.java b/presto-main/src/main/java/io/prestosql/operator/HashBuilderGroupJoinOperatorFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..46613f989267f3f6d508094fb26dd510a20449e8 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/HashBuilderGroupJoinOperatorFactory.java @@ -0,0 +1,312 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator; + +import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; +import io.prestosql.execution.Lifespan; +import io.prestosql.operator.aggregation.AccumulatorFactory; +import io.prestosql.operator.aggregation.partial.PartialAggregationController; +import io.prestosql.operator.groupjoin.ExecutionHelperFactory; +import io.prestosql.spi.plan.AggregationNode; +import io.prestosql.spi.plan.PlanNodeId; +import io.prestosql.spi.type.Type; +import io.prestosql.sql.gen.JoinCompiler; +import io.prestosql.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Verify.verify; +import static java.util.Objects.requireNonNull; + +public class HashBuilderGroupJoinOperatorFactory + implements OperatorFactory +{ + private final int operatorId; + private final PlanNodeId planNodeId; + private ExecutionHelperFactory executionHelperFactory; + private final JoinBridgeManager lookupSourceFactoryManager; + private final List outputChannels; + private final List hashChannels; + private final OptionalInt preComputedHashChannel; + private final Optional filterFunctionFactory; + private final Optional sortChannel; + private final Optional countChannel; + private final List searchFunctionFactories; + private final PagesIndex.Factory pagesIndexFactory; + private final int expectedPositions; + private final boolean spillEnabled; + private final Map partitionIndexManager = new HashMap<>(); + private boolean closed; + private boolean spillToHdfsEnabled; + private final GroupJoinAggregator aggrOnAggrfactory; + private final GroupJoinAggregator aggrfactory; + + public static Builder builder() + { + return new Builder(); + } + + public HashBuilderGroupJoinOperatorFactory( + int operatorId, + PlanNodeId planNodeId, + JoinBridgeManager lookupSourceFactoryManager, + List outputChannels, + List hashChannels, + OptionalInt preComputedHashChannel, + Optional filterFunctionFactory, + Optional sortChannel, + Optional countChannel, + List searchFunctionFactories, + int expectedPositions, + PagesIndex.Factory pagesIndexFactory, + boolean spillEnabled, + boolean spillToHdfsEnabled, + GroupJoinAggregator aggrfactory, + GroupJoinAggregator aggrOnAggrfactory, + ExecutionHelperFactory executionHelperFactory) + { + this.operatorId = operatorId; + this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); + this.executionHelperFactory = requireNonNull(executionHelperFactory, "executionHelperFactory is null"); + requireNonNull(sortChannel, "sortChannel can not be null"); + requireNonNull(searchFunctionFactories, "searchFunctionFactories is null"); + checkArgument(sortChannel.isPresent() != searchFunctionFactories.isEmpty(), "both or none sortChannel and searchFunctionFactories must be set"); + this.lookupSourceFactoryManager = requireNonNull(lookupSourceFactoryManager, "lookupSourceFactoryManager is null"); + + this.outputChannels = ImmutableList.copyOf(requireNonNull(outputChannels, "outputChannels is null")); + this.hashChannels = ImmutableList.copyOf(requireNonNull(hashChannels, "hashChannels is null")); + this.preComputedHashChannel = requireNonNull(preComputedHashChannel, "preComputedHashChannel is null"); + this.filterFunctionFactory = requireNonNull(filterFunctionFactory, "filterFunctionFactory is null"); + this.sortChannel = sortChannel; + this.countChannel = countChannel; + this.searchFunctionFactories = ImmutableList.copyOf(searchFunctionFactories); + this.pagesIndexFactory = requireNonNull(pagesIndexFactory, "pagesIndexFactory is null"); + this.spillEnabled = spillEnabled; + this.spillToHdfsEnabled = spillToHdfsEnabled; + this.expectedPositions = expectedPositions; + + this.aggrfactory = aggrfactory; + this.aggrOnAggrfactory = aggrOnAggrfactory; + } + + @Override + public Operator createOperator(DriverContext driverContext) + { + checkState(!closed, "Factory is already closed"); + OperatorContext addOperatorContext = driverContext.addOperatorContext(operatorId, planNodeId, HashBuilderGroupJoinOperator.class.getSimpleName()); + + PartitionedLookupSourceFactory partitionedLookupSourceFactory = this.lookupSourceFactoryManager.getJoinBridge(driverContext.getLifespan()); + int incrementPartitionIndex = getAndIncrementPartitionIndex(driverContext.getLifespan()); + // Snapshot: make driver ID and source/partition index the same, to ensure consistency before and after resuming. + // LocalExchangeSourceOperator also uses the same mechanism to ensure consistency. + if (addOperatorContext.isSnapshotEnabled()) { + incrementPartitionIndex = driverContext.getDriverId(); + } + verify(incrementPartitionIndex < partitionedLookupSourceFactory.partitions()); + return new HashBuilderGroupJoinOperator( + addOperatorContext, + partitionedLookupSourceFactory, + incrementPartitionIndex, + outputChannels, + hashChannels, + preComputedHashChannel, + filterFunctionFactory, + sortChannel, + countChannel, + searchFunctionFactories, + expectedPositions, + pagesIndexFactory, + spillEnabled, + spillToHdfsEnabled, + aggrfactory, + aggrOnAggrfactory, + executionHelperFactory); + } + + @Override + public void noMoreOperators() + { + closed = true; + } + + @Override + public OperatorFactory duplicate() + { + throw new UnsupportedOperationException("Parallel hash build can not be duplicated"); + } + + private int getAndIncrementPartitionIndex(Lifespan lifespan) + { + return partitionIndexManager.compute(lifespan, (k, v) -> v == null ? 1 : v + 1) - 1; + } + + public static class Builder + { + private GroupJoinAggregator aggrOnAggrfactory; + private GroupJoinAggregator aggrfactory; + private int operatorId; + private PlanNodeId planNodeId; + private JoinBridgeManager lookupSourceFactoryManager; + private List outputChannels; + private List hashChannels; + private OptionalInt preComputedHashChannel; + private Optional filterFunctionFactory; + private Optional sortChannel; + private Optional countChannel; + private List searchFunctionFactories; + private int expectedPositions; + private PagesIndex.Factory pagesIndexFactory; + private boolean spillEnabled; + private boolean spillToHdfsEnabled; + private ExecutionHelperFactory executionHelperFactory; + + public Builder() + { + } + + public Builder withExecutorHelperFactory(ExecutionHelperFactory executionHelperFactory) + { + this.executionHelperFactory = requireNonNull(executionHelperFactory, "executionHelperFactory is null"); + return this; + } + + public Builder withJoinInfo(int operatorId, + PlanNodeId planNodeId, + JoinBridgeManager lookupSourceFactoryManager, + List outputChannels, + List hashChannels, + OptionalInt preComputedHashChannel, + Optional filterFunctionFactory, + Optional sortChannel, + Optional countChannel, + List searchFunctionFactories, + int expectedPositions, + PagesIndex.Factory pagesIndexFactory, + boolean spillEnabled, + boolean spillToHdfsEnabled) + { + this.operatorId = operatorId; + this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); + requireNonNull(sortChannel, "sortChannel can not be null"); + requireNonNull(searchFunctionFactories, "searchFunctionFactories is null"); + checkArgument(sortChannel.isPresent() != searchFunctionFactories.isEmpty(), "both or none sortChannel and searchFunctionFactories must be set"); + this.lookupSourceFactoryManager = requireNonNull(lookupSourceFactoryManager, "lookupSourceFactoryManager is null"); + + this.outputChannels = ImmutableList.copyOf(requireNonNull(outputChannels, "outputChannels is null")); + this.hashChannels = ImmutableList.copyOf(requireNonNull(hashChannels, "hashChannels is null")); + this.preComputedHashChannel = requireNonNull(preComputedHashChannel, "preComputedHashChannel is null"); + this.filterFunctionFactory = requireNonNull(filterFunctionFactory, "filterFunctionFactory is null"); + this.sortChannel = sortChannel; + this.countChannel = countChannel; + this.searchFunctionFactories = ImmutableList.copyOf(searchFunctionFactories); + this.pagesIndexFactory = requireNonNull(pagesIndexFactory, "pagesIndexFactory is null"); + this.spillEnabled = spillEnabled; + this.spillToHdfsEnabled = spillToHdfsEnabled; + this.expectedPositions = expectedPositions; + return this; + } + + public Builder withAggrOnAggrFactory(List groupByTypes, + List groupByChannels, + List globalAggregationGroupIds, + AggregationNode.Step step, + List accumulatorFactories, + Optional hashChannel, + Optional groupIdChannel, + int expectedGroups, + Optional maxPartialMemory, + JoinCompiler joinCompiler, + boolean useSystemMemory, + Optional partialAggregationController, + boolean produceDefaultOutput) + { + aggrOnAggrfactory = GroupJoinAggregator.buildGroupJoinAggregator(groupByTypes, + groupByChannels, + globalAggregationGroupIds, + step, + accumulatorFactories, + hashChannel, + groupIdChannel, + expectedGroups, + maxPartialMemory, + joinCompiler, + useSystemMemory, + partialAggregationController, + produceDefaultOutput); + return this; + } + + public Builder withAggrFactory(List groupByTypes, + List groupByChannels, + List globalAggregationGroupIds, + AggregationNode.Step step, + List accumulatorFactories, + Optional hashChannel, + Optional groupIdChannel, + int expectedGroups, + Optional maxPartialMemory, + JoinCompiler joinCompiler, + boolean useSystemMemory, + Optional partialAggregationController, + boolean produceDefaultOutput) + { + aggrfactory = GroupJoinAggregator.buildGroupJoinAggregator(groupByTypes, + groupByChannels, + globalAggregationGroupIds, + step, + accumulatorFactories, + hashChannel, + groupIdChannel, + expectedGroups, + maxPartialMemory, + joinCompiler, + useSystemMemory, + partialAggregationController, + produceDefaultOutput); + return this; + } + + public HashBuilderGroupJoinOperatorFactory build() + { + requireNonNull(aggrfactory, "aggrfactory is null"); + requireNonNull(aggrOnAggrfactory, "aggrOnAggrfactory is null"); + requireNonNull(hashChannels, "hashChannels is null"); + return new HashBuilderGroupJoinOperatorFactory( + operatorId, + planNodeId, + lookupSourceFactoryManager, + outputChannels, + hashChannels, + preComputedHashChannel, + filterFunctionFactory, + sortChannel, + countChannel, + searchFunctionFactories, + expectedPositions, + pagesIndexFactory, + spillEnabled, + spillToHdfsEnabled, + aggrfactory, + aggrOnAggrfactory, + executionHelperFactory); + } + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/IPagesHash.java b/presto-main/src/main/java/io/prestosql/operator/IPagesHash.java index 5b5306829e4cc8131f81b7d6d5d41964eaafdb32..7734c6667c5831ef8ac53b4d35bab0a2d4b68cba 100644 --- a/presto-main/src/main/java/io/prestosql/operator/IPagesHash.java +++ b/presto-main/src/main/java/io/prestosql/operator/IPagesHash.java @@ -13,6 +13,7 @@ */ package io.prestosql.operator; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; @@ -50,6 +51,16 @@ public interface IPagesHash return result; } + default long getCountForJoinPosition(long position, int channel) + { + throw new UnsupportedOperationException("Only supported for Group Join usage"); + } + + default AggregationBuilder getAggregationBuilder() + { + throw new UnsupportedOperationException("Only supported for Group Join usage"); + } + void appendTo(long position, PageBuilder pageBuilder, int outputChannelOffset); default int getHashPosition(long raw, long mask) diff --git a/presto-main/src/main/java/io/prestosql/operator/JoinHash.java b/presto-main/src/main/java/io/prestosql/operator/JoinHash.java index 8c96cb6721e7cec3f34d5b96471987a0e42f1bbe..ada206b207cb734486551f5c76b8065bd6931a53 100644 --- a/presto-main/src/main/java/io/prestosql/operator/JoinHash.java +++ b/presto-main/src/main/java/io/prestosql/operator/JoinHash.java @@ -13,6 +13,7 @@ */ package io.prestosql.operator; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; import org.openjdk.jol.info.ClassLayout; @@ -119,6 +120,18 @@ public final class JoinHash return filterFunction == null || filterFunction.filter(toIntExact(currentJoinPosition), probePosition, allProbeChannelsPage); } + @Override + public long getCountForJoinPosition(long position, int channel) + { + return pagesHash.getCountForJoinPosition(position, channel); + } + + @Override + public AggregationBuilder getAggregationBuilder() + { + return pagesHash.getAggregationBuilder(); + } + @Override public void appendTo(long position, PageBuilder pageBuilder, int outputChannelOffset) { diff --git a/presto-main/src/main/java/io/prestosql/operator/JoinUtils.java b/presto-main/src/main/java/io/prestosql/operator/JoinUtils.java index 6689b44498ed41b7848816cb2ab84fcc2fb44548..bf6a7de6b302f4136769dfd3338749c8b584f3eb 100644 --- a/presto-main/src/main/java/io/prestosql/operator/JoinUtils.java +++ b/presto-main/src/main/java/io/prestosql/operator/JoinUtils.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableList; import io.prestosql.spi.Page; import io.prestosql.spi.block.Block; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.ProjectNode; import io.prestosql.sql.planner.optimizations.PlanNodeSearcher; @@ -72,6 +73,15 @@ public final class JoinUtils .where(joinNode -> isRemoteReplicatedExchange(joinNode) || isRemoteReplicatedSourceNode(joinNode)) .matches(); } + else if (node instanceof JoinOnAggregationNode) { + return PlanNodeSearcher.searchFrom(((JoinOnAggregationNode) node).getRight()) + .recurseOnlyWhen( + MorePredicates.isInstanceOfAny(ProjectNode.class) + .or(JoinUtils::isLocalRepartitionExchange) + .or(JoinUtils::isLocalGatherExchange)) // used in cross join case + .where(joinNode -> isRemoteReplicatedExchange(joinNode) || isRemoteReplicatedSourceNode(joinNode)) + .matches(); + } return PlanNodeSearcher.searchFrom(((SemiJoinNode) node).getFilteringSource()) .recurseOnlyWhen( MorePredicates.isInstanceOfAny(ProjectNode.class) diff --git a/presto-main/src/main/java/io/prestosql/operator/LookupGroupJoinOperator.java b/presto-main/src/main/java/io/prestosql/operator/LookupGroupJoinOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..6740dc1ff0ca347bbda5c695ab4334aed6ba57f9 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/LookupGroupJoinOperator.java @@ -0,0 +1,861 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Closer; +import com.google.common.util.concurrent.ListenableFuture; +import io.airlift.log.Logger; +import io.prestosql.memory.context.LocalMemoryContext; +import io.prestosql.operator.GroupJoinProbe.GroupJoinProbeFactory; +import io.prestosql.operator.LookupJoinOperator.SpillInfoSnapshot; +import io.prestosql.operator.LookupJoinOperators.JoinType; +import io.prestosql.operator.PartitionedConsumption.Partition; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; +import io.prestosql.operator.aggregation.builder.InMemoryHashAggregationBuilder; +import io.prestosql.operator.aggregation.builder.InMemoryHashAggregationBuilderWithReset; +import io.prestosql.operator.exchange.LocalPartitionGenerator; +import io.prestosql.operator.groupjoin.ExecutionHelperFactory; +import io.prestosql.snapshot.SingleInputSnapshotState; +import io.prestosql.spi.Page; +import io.prestosql.spi.type.Type; +import io.prestosql.spiller.PartitioningSpiller; +import io.prestosql.spiller.PartitioningSpillerFactory; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.Supplier; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Verify.verify; +import static io.airlift.concurrent.MoreFutures.addSuccessCallback; +import static io.airlift.concurrent.MoreFutures.getDone; +import static io.prestosql.SystemSessionProperties.isInnerJoinSpillFilteringEnabled; +import static io.prestosql.operator.LookupJoinOperators.JoinType.FULL_OUTER; +import static io.prestosql.operator.LookupJoinOperators.JoinType.PROBE_OUTER; +import static java.util.Collections.emptyIterator; +import static java.util.Objects.requireNonNull; + +public class LookupGroupJoinOperator + implements Operator +{ + @VisibleForTesting + public enum State + { + /** + * Operator accepts input + */ + CONSUMING_INPUT, + + /** + * All inputs accepted, finishing aggregations + */ + AGGR_FINISHING, + + /** + * Aggregation on input finished + */ + AGGR_FINISHED, + + /** + * LookupSource has been built and passed on without any spill occurring + */ + SOURCE_BUILT, + + /** + * No longer needed + */ + CLOSED + } + + private final Logger log = Logger.get(LookupGroupJoinOperator.class); + + private final OperatorContext operatorContext; + private ExecutionHelperFactory executionHelperFactory; + + private final GroupJoinProbeFactory joinProbeFactory; + private final Runnable afterClose; + + private final PartitioningSpillerFactory partitioningSpillerFactory; + private Runnable afterMemOpFinish; + private final OptionalInt lookupJoinsCount; + private final HashGenerator hashGenerator; + private final LookupSourceFactory lookupSourceFactory; + + private final JoinStatisticsCounter statisticsCounter; + + private final LookupGroupJoinPageBuilder pageBuilder; + + private final boolean probeOnOuterSide; + private final boolean spillBypassEnabled; + + private final ListenableFuture lookupSourceProviderFuture; + private LookupSourceProvider lookupSourceProvider; + private GroupJoinProbe probe; + + private Page outputPage; + + private Optional spiller = Optional.empty(); + private Optional partitionGenerator = Optional.empty(); + private ListenableFuture spillInProgress = NOT_BLOCKED; + private long inputPageSpillEpoch; + private boolean closed; + private boolean finishing; + private boolean unspilling; + private boolean finished; + private long joinPosition = -1; + private int joinSourcePositions; + + private boolean currentProbePositionProducedRow; + + @Nullable + private ListenableFuture>> partitionedConsumption; + @Nullable + private Iterator>> lookupPartitions; + private Optional>> currentPartition = Optional.empty(); + private Optional>> unspilledLookupSource = Optional.empty(); + private Iterator unspilledInputPages = emptyIterator(); + private Iterator unspilledMemoryPartitions = emptyIterator(); + private Map> backUpUnspilledMemoryPartitions = new HashMap<>(); + private Map backUpRestoredPages = new HashMap<>(); + private Integer restoredPartition; + private final GroupJoinAggregator aggregator; + private final GroupJoinAggregator aggrOnAggregator; + + private final SingleInputSnapshotState snapshotState; + private boolean isSingleSessionSpiller; + private List spilledPartitionsList = new ArrayList<>(); + + protected AggregationBuilder aggregationBuilder; + protected AggregationBuilder probeAggrOnAggregationBuilder; + protected AggregationBuilder buildAggrOnAggregationBuilder; + protected LocalMemoryContext aggrMemoryContext; + protected LocalMemoryContext aggrOnAggrMemoryContext; + + private final LocalMemoryContext localUserMemoryContext; + private final LocalMemoryContext localRevocableMemoryContext; + + private final HashCollisionsCounter hashCollisionsCounter; + protected long numberOfInputRowsProcessed; + protected long numberOfUniqueRowsProduced; + protected Work unfinishedAggrWork; + protected boolean aggregationInputProcessed; + private final boolean spillEnabled = false; + protected WorkProcessor aggrOutputPages; + protected State state = State.CONSUMING_INPUT; + private ListenableFuture executionHelper = NOT_BLOCKED; + private final PagesIndex index; + private Iterator pageIndexItr; + + public LookupGroupJoinOperator( + OperatorContext operatorContext, + boolean forked, + List probeTypes, + List outputTypes, + List buildTypes, + JoinType joinType, + LookupSourceFactory lookupSourceFactory, + GroupJoinProbeFactory joinProbeFactory, + Runnable afterClose, + OptionalInt lookupJoinsCount, + HashGenerator hashGenerator, + PartitioningSpillerFactory partitioningSpillerFactory, + Runnable afterMemOpFinish, + boolean isSingleSessionSpiller, + GroupJoinAggregator aggregator, + GroupJoinAggregator aggrOnAggregator, + List probeFinalOutputChannels, + List buildFinalOutputChannels, + ExecutionHelperFactory executionHelperFactory, + int expectedPositions, + PagesIndex.Factory pagesIndexFactory) + { + this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); + this.executionHelperFactory = executionHelperFactory; + List probeTypes1 = ImmutableList.copyOf(requireNonNull(probeTypes, "probeTypes is null")); + + requireNonNull(joinType, "joinType is null"); + // Cannot use switch case here, because javac will synthesize an inner class and cause IllegalAccessError + probeOnOuterSide = joinType == PROBE_OUTER || joinType == FULL_OUTER; + spillBypassEnabled = probeOnOuterSide || !isInnerJoinSpillFilteringEnabled(operatorContext.getDriverContext().getSession()); + + this.joinProbeFactory = requireNonNull(joinProbeFactory, "joinProbeFactory is null"); + this.afterClose = requireNonNull(afterClose, "afterClose is null"); + this.lookupJoinsCount = requireNonNull(lookupJoinsCount, "lookupJoinsCount is null"); + this.hashGenerator = requireNonNull(hashGenerator, "hashGenerator is null"); + this.lookupSourceFactory = requireNonNull(lookupSourceFactory, "lookupSourceFactory is null"); + this.partitioningSpillerFactory = requireNonNull(partitioningSpillerFactory, "partitioningSpillerFactory is null"); + this.lookupSourceProviderFuture = lookupSourceFactory.createLookupSourceProvider(); + + this.statisticsCounter = new JoinStatisticsCounter(joinType); + operatorContext.setInfoSupplier(this.statisticsCounter); + + this.pageBuilder = new LookupGroupJoinPageBuilder(outputTypes, buildTypes, buildFinalOutputChannels, probeFinalOutputChannels); + this.snapshotState = operatorContext.isSnapshotEnabled() ? SingleInputSnapshotState.forOperator(this, operatorContext) : null; + + this.afterMemOpFinish = afterMemOpFinish; + this.isSingleSessionSpiller = isSingleSessionSpiller; + + this.hashCollisionsCounter = new HashCollisionsCounter(operatorContext); + operatorContext.setInfoSupplier(hashCollisionsCounter); + + this.aggrMemoryContext = operatorContext.localUserMemoryContext(); + this.aggrOnAggrMemoryContext = operatorContext.localUserMemoryContext(); + if (aggregator.isUseSystemMemory()) { + this.aggrMemoryContext = operatorContext.localSystemMemoryContext(); + } + if (aggrOnAggregator.isUseSystemMemory()) { + this.aggrOnAggrMemoryContext = operatorContext.localSystemMemoryContext(); + } + + this.aggregator = aggregator; + this.aggrOnAggregator = aggrOnAggregator; + requireNonNull(pagesIndexFactory, "pagesIndexFactory is null"); + this.index = pagesIndexFactory.newPagesIndex(probeTypes, expectedPositions); + this.localUserMemoryContext = operatorContext.localUserMemoryContext(); + this.localRevocableMemoryContext = operatorContext.localRevocableMemoryContext(); + createAggrOnAggregationBuilder(); + } + + @Override + public OperatorContext getOperatorContext() + { + return operatorContext; + } + + @Override + public void finish() + { + if (finishing) { + return; + } + if (State.CONSUMING_INPUT == state) { + log.info("State changed to %s", State.AGGR_FINISHING); + state = State.AGGR_FINISHING; + } + finishing = true; + } + + @Override + public boolean isFinished() + { + boolean finishedNow = this.finishing && this.finished && probe == null && pageBuilder.isEmpty() && outputPage == null; + // if finishedNow drop references so memory is freed early + if (finishedNow) { + close(); + } + return finishedNow; + } + + @Override + public ListenableFuture isBlocked() + { + if (state == State.CONSUMING_INPUT || state == State.AGGR_FINISHING) { + if (executionHelper != null) { + return executionHelper.isDone() ? NOT_BLOCKED : executionHelper; + } + return NOT_BLOCKED; + } + /*if (finishing) { + return NOT_BLOCKED; + }*/ + + return lookupSourceProviderFuture; + } + + @Override + public boolean needsInput() + { + if (state == State.CONSUMING_INPUT) { + if (aggrOutputPages != null) { + return false; + } + else if (aggregationBuilder != null && aggregationBuilder.isFull()) { + return false; + } + else { + // TODO Vineet Need to move this out of needsInput and need to make it light weight. + if (unfinishedAggrWork != null) { + if (executionHelper != null) { + checkAndResetExeHelper(); + return false; + } + executionHelper = executionHelperFactory.create().submitWork(() -> { + boolean workDone = unfinishedAggrWork.process(); + aggregationBuilder.updateMemory(); + while (!workDone) { + workDone = unfinishedAggrWork.process(); + aggregationBuilder.updateMemory(); + } + + unfinishedAggrWork = null; + executionHelper = null; + }); + return false; + } + return true; + } + } + return false; + } + + private void checkAndResetExeHelper() + { + if (executionHelper.isDone()) { + executionHelper = null; + } + } + + @Override + public void addInput(Page page) + { + requireNonNull(page, "page is null"); + //checkState(probe == null, "Current page has not been completely processed yet"); + //checkState(tryFetchLookupSourceProvider(), "Not ready to handle input yet"); + // create Aggregators and pass page to them for process + checkState(state == State.CONSUMING_INPUT, "Operator is already finishing"); + aggregationInputProcessed = true; + if (aggregationBuilder == null) { + createAggregationBuilder(); + } + else { + checkState(!aggregationBuilder.isFull(), "Aggregation buffer is full"); + } + + // process the current page; save the unfinished work if we are waiting for memory + unfinishedAggrWork = aggregationBuilder.processPage(page); + if (unfinishedAggrWork.process()) { + unfinishedAggrWork = null; + } + aggregationBuilder.updateMemory(); + numberOfInputRowsProcessed += page.getPositionCount(); + } + + private void createProbe(Page page) + { + // create probe + if (buildAggrOnAggregationBuilder == null) { + LookupSource lookupSource = lookupSourceProvider.withLease((lookupSourceLease -> lookupSourceLease.getLookupSource())); + buildAggrOnAggregationBuilder = lookupSource.getAggregationBuilder().duplicate(); + } + probe = joinProbeFactory.createGroupJoinProbe(page, false, lookupSourceProvider, probeAggrOnAggregationBuilder, buildAggrOnAggregationBuilder); + + // initialize to invalid join position to force output code to advance the cursors + joinPosition = -1; + } + + private boolean tryFetchLookupSourceProvider() + { + if (lookupSourceProvider == null) { + if (!lookupSourceProviderFuture.isDone()) { + return false; + } + lookupSourceProvider = requireNonNull(getDone(lookupSourceProviderFuture)); + statisticsCounter.updateLookupSourcePositions(lookupSourceProvider.withLease(lookupSourceLease -> lookupSourceLease.getLookupSource().getJoinPositionCount())); + } + return true; + } + + private void updateIndex(Page page) + { + index.addPage(page); + if (spillEnabled) { + localRevocableMemoryContext.setBytes(index.getEstimatedSize().toBytes()); + } + else { + if (!localUserMemoryContext.trySetBytes(index.getEstimatedSize().toBytes())) { + index.compact(); + localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes()); + } + } + } + + private void finishAggregation() + { + DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); + Page page = processAggregation(); + while (page != null) { + updateIndex(page); + if (yieldSignal.isSet()) { + break; + } + page = processAggregation(); + } + } + + @Override + public Page getOutput() + { + switch (state) { + case CONSUMING_INPUT: + // Only prepare the Aggr outputs. + if (aggregationBuilder != null && aggregationBuilder.isFull()) { + finishAggregation(); + } + if (lookupSourceProviderFuture.isDone()) { + // eager processing of aggregated pages as build side is done. + createConditionalPageItrAndProbe(); + break; + } + return null; + case AGGR_FINISHING: + // Prepare the Aggr outputs for pending unprocessed pages. + finishAggregation(); + if (lookupSourceProviderFuture.isDone()) { + // eager processing of aggregated pages as build side is done. + createConditionalPageItrAndProbe(); + break; + } + return null; + case AGGR_FINISHED: + // All Aggregation done, probe can start + if (lookupSourceProviderFuture.isDone()) { + createConditionalPageItrAndProbe(); + } + // Terminal Condition. + if (finishing && probe == null && tryFetchLookupSourceProvider() && pageIndexItr != null && !pageIndexItr.hasNext()) { + log.info("State changed to %s", State.SOURCE_BUILT); + state = State.SOURCE_BUILT; + } + /*if (probe != null) { + break; + }*/ + break; + case SOURCE_BUILT: + break; + case CLOSED: + // no-op + return null; + } + + /*if (probe == null && pageBuilder.isEmpty()*//* && !finishing*//*) { + return null; + }*/ + if (!lookupSourceProviderFuture.isDone()) { + return null; + } + + if (!tryFetchLookupSourceProvider()) { + if (!finishing) { + return null; + } + + verify(finishing); + // We are no longer interested in the build side (the lookupSourceProviderFuture's value). + addSuccessCallback(lookupSourceProviderFuture, LookupSourceProvider::close); + lookupSourceProvider = new StaticLookupSourceProvider(new EmptyLookupSource()); + } + + if (probe == null && outputPage == null && state == State.SOURCE_BUILT) { + /* + * We do not have input probe and we won't have any, as we're finishing. + * Let LookupSourceFactory know LookupSources can be disposed as far as we're concerned. + */ + verify(partitionedConsumption == null, "partitioned consumption already started"); + lookupSourceProvider.close(); + partitionedConsumption = lookupSourceFactory.finishProbeOperator(lookupJoinsCount); + afterMemOpFinish.run(); + afterMemOpFinish = () -> {}; + //unspilling = true; + finished = true; + } + + if (probe != null) { + processProbe(); + } + + if (outputPage != null) { + verify(pageBuilder.isEmpty()); + Page output = outputPage; + outputPage = null; + return output; + } + + // It is impossible to have probe == null && !pageBuilder.isEmpty(), + // because we will flush a page whenever we reach the probe end + verify(probe != null || pageBuilder.isEmpty()); + return null; + } + + private void createConditionalPageItrAndProbe() + { + if (pageIndexItr == null) { + log.info("Lookup Probe Page Itr created"); + pageIndexItr = index.getPagesIterator(); + } + if (pageIndexItr != null && finishing && state == State.AGGR_FINISHED) { + if (pageIndexItr instanceof PagesIndex.PageIterator) { + ((PagesIndex.PageIterator) pageIndexItr).setFinished(true); + } + } + if (probe == null && outputPage == null && tryFetchLookupSourceProvider() && pageIndexItr != null && pageIndexItr.hasNext()) { + Page page = pageIndexItr.next(); + if (page != null) { + createProbe(page); + } + } + } + + protected boolean hasOrderBy() + { + return aggregator.hasOrderBy(); + } + + protected boolean hasDistinct() + { + return aggregator.hasDistinct(); + } + + public void createAggregationBuilder() + { + if (aggregator.getStep().isOutputPartial() || !spillEnabled || hasOrderBy() || hasDistinct()) { + aggregationBuilder = new InMemoryHashAggregationBuilder( + aggregator.getAccumulatorFactories(), + aggregator.getStep(), + aggregator.getExpectedGroups(), + aggregator.getGroupByTypes(), + aggregator.getGroupByChannels(), + aggregator.getHashChannel(), + operatorContext, + aggregator.getMaxPartialMemory(), + aggregator.getJoinCompiler(), + () -> { + aggrMemoryContext.setBytes(((InMemoryHashAggregationBuilder) aggregationBuilder).getSizeInMemory()); + if (aggregator.getStep().isOutputPartial() && aggregator.getMaxPartialMemory().isPresent()) { + // do not yield on memory for partial aggregations + return true; + } + return operatorContext.isWaitingForMemory().isDone(); + }); + } + else { + throw new UnsupportedOperationException("Not Supported"); + } + } + + public void createAggrOnAggregationBuilder() + { + if (aggrOnAggregator.getStep().isOutputPartial() || !spillEnabled || hasOrderBy() || hasDistinct()) { + probeAggrOnAggregationBuilder = new InMemoryHashAggregationBuilderWithReset( + aggrOnAggregator.getAccumulatorFactories(), + aggrOnAggregator.getStep(), + aggrOnAggregator.getExpectedGroups(), + aggrOnAggregator.getGroupByTypes(), + aggrOnAggregator.getGroupByChannels(), + aggrOnAggregator.getHashChannel(), + operatorContext, + aggrOnAggregator.getMaxPartialMemory(), + aggrOnAggregator.getJoinCompiler(), + () -> { + aggrOnAggrMemoryContext.setBytes(((InMemoryHashAggregationBuilder) probeAggrOnAggregationBuilder).getSizeInMemory()); + if (aggrOnAggregator.getStep().isOutputPartial() && aggrOnAggregator.getMaxPartialMemory().isPresent()) { + // do not yield on memory for partial aggregations + return true; + } + return operatorContext.isWaitingForMemory().isDone(); + }); + } + else { + throw new UnsupportedOperationException("Not Supported"); + } + } + + public Page processAggregation() + { + if (state == State.AGGR_FINISHED) { + return null; + } + + // process unfinished work if one exists + if (unfinishedAggrWork != null) { + boolean workDone = unfinishedAggrWork.process(); + aggregationBuilder.updateMemory(); + if (!workDone) { + return null; + } + unfinishedAggrWork = null; + } + + if (aggrOutputPages == null) { + if (!aggregationInputProcessed && aggregator.isProduceDefaultOutput()) { + // global aggregations always generate an output row with the default aggregation output (e.g. 0 for COUNT, NULL for SUM) + if (state == State.AGGR_FINISHING) { + log.info("State changed to %s", State.AGGR_FINISHED); + state = State.AGGR_FINISHED; + } + return aggregator.getGlobalAggregationOutput(); + } + + if (aggregationBuilder == null) { + if (state == State.AGGR_FINISHING) { + log.info("State changed to %s", State.AGGR_FINISHED); + state = State.AGGR_FINISHED; + } + return null; + } + + // Only flush if we are finishing(consuming input will change to AggrFinishing) or the aggregation builder is full + if (state == State.CONSUMING_INPUT && !aggregationBuilder.isFull()) { + return null; + } + + aggrOutputPages = aggregationBuilder.buildResult(); + } + + if (!aggrOutputPages.process()) { + return null; + } + + if (aggrOutputPages.isFinished()) { + closeAggregationBuilder(); + return null; + } + + Page result = aggrOutputPages.getResult(); + numberOfUniqueRowsProduced += result.getPositionCount(); + return result; + } + + protected void closeAggregationBuilder() + { + aggrOutputPages = null; + if (aggregationBuilder != null) { + aggregationBuilder.recordHashCollisions(hashCollisionsCounter); + aggregationBuilder.close(); + // aggregationBuilder.close() will release all memory reserved in memory accounting. + // The reference must be set to null afterwards to avoid unaccounted memory. + aggregationBuilder = null; + } + aggrMemoryContext.setBytes(0); + aggregator.getPartialAggregationController().ifPresent( + controller -> controller.onFlush(numberOfInputRowsProcessed, numberOfUniqueRowsProduced)); + numberOfInputRowsProcessed = 0; + numberOfUniqueRowsProduced = 0; + } + + protected void closeProbeAggrOnAggregationBuilder() + { + if (probeAggrOnAggregationBuilder != null) { + probeAggrOnAggregationBuilder.recordHashCollisions(hashCollisionsCounter); + probeAggrOnAggregationBuilder.close(); + // probeAggregationBuilder.close() will release all memory reserved in memory accounting. + // The reference must be set to null afterwards to avoid unaccounted memory. + probeAggrOnAggregationBuilder = null; + } + aggrOnAggrMemoryContext.setBytes(0); + } + + private void processProbe() + { + verify(probe != null); + Optional value = lookupSourceProvider.withLease(lookupSourceLease -> { + if (lookupSourceLease.spillEpoch() == inputPageSpillEpoch) { + // Spill state didn't change, so process as usual. + processProbe(lookupSourceLease.getLookupSource()); + return Optional.empty(); + } + + return Optional.of(SpillInfoSnapshot.from(lookupSourceLease)); + }); + if (!value.isPresent()) { + return; + } + /*long joinPositionWithinPartition; + if (joinPosition >= 0) { + joinPositionWithinPartition = lookupSourceProvider.withLease(lookupSourceLease -> lookupSourceLease.getLookupSource().joinPositionWithinPartition(joinPosition)); + } + else { + joinPositionWithinPartition = -1; + }*/ + if (probe == null) { + return; + } + + Page currentPage = probe.getPage(); + int currentPosition = probe.getPosition(); + long currentJoinPosition = this.joinPosition; + boolean probePositionProducedRow = this.currentProbePositionProducedRow; + clearProbe(); + if (currentPosition < 0) { + // Processing of the page hasn't been started yet. + createProbe(currentPage); + } + else { + Page remaining = pageTail(currentPage, currentPosition); + restoreProbe(remaining, currentJoinPosition, probePositionProducedRow, joinSourcePositions); + } + } + + private void processProbe(LookupSource lookupSource) + { + verify(probe != null); + DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); + while (!yieldSignal.isSet()) { + if (probe.getPosition() >= 0) { + if (!joinCurrentPosition(lookupSource, yieldSignal)) { + break; + } + /*if (!currentProbePositionProducedRow) { + currentProbePositionProducedRow = true; + }*/ + } + currentProbePositionProducedRow = false; + if (!advanceProbePosition(lookupSource)) { + break; + } + statisticsCounter.recordProbe(joinSourcePositions); + joinSourcePositions = 0; + } + } + + private void restoreProbe(Page probePage, long joinPosition, boolean currentProbePositionProducedRow, int joinSourcePositions) + { + verify(probe == null); + createProbe(probePage); + if (probe != null) { + verify(probe.advanceNextPosition()); + } + this.joinPosition = joinPosition; + this.currentProbePositionProducedRow = currentProbePositionProducedRow; + this.joinSourcePositions = joinSourcePositions; + } + + private Page pageTail(Page currentPage, int startAtPosition) + { + verify(currentPage.getPositionCount() - startAtPosition >= 0); + return currentPage.getRegion(startAtPosition, currentPage.getPositionCount() - startAtPosition); + } + + @Override + public void close() + { + if (closed) { + return; + } + closed = true; + probe = null; + if (state == State.CONSUMING_INPUT) { + closeAggregationBuilder(); + } + + try (Closer closer = Closer.create()) { + // `afterClose` must be run last. + // Closer is documented to mimic try-with-resource, which implies close will happen in reverse order. + closer.register(index::clear); + closer.register(afterMemOpFinish::run); + closer.register(afterClose::run); + + closer.register(pageBuilder::reset); + closer.register(() -> Optional.ofNullable(lookupSourceProvider).ifPresent(LookupSourceProvider::close)); + closer.register(() -> { + if (snapshotState != null) { + snapshotState.close(); + } + }); + spiller.ifPresent(closer::register); + closer.register(this::closeProbeAggrOnAggregationBuilder); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Produce rows matching join condition for the current probe position. If this method was called previously + * for the current probe position, calling this again will produce rows that wasn't been produced in previous + * invocations. + * + * @return true if all eligible rows have been produced; false otherwise + */ + private boolean joinCurrentPosition(LookupSource lookupSource, DriverYieldSignal yieldSignal) + { + // while we have a position on lookup side to join against... + while (joinPosition >= 0) { + if (lookupSource.isJoinPositionEligible(joinPosition, probe.getPosition(), probe.getPage())) { + currentProbePositionProducedRow = true; + + // Build count * Probe Rec, probe count * Build Rec, then add to Output Page Builder + pageBuilder.appendRow(probe, lookupSource, joinPosition); + joinSourcePositions++; + } + + // get next position on lookup side for this probe row + joinPosition = lookupSource.getNextJoinPosition(joinPosition, probe.getPosition(), probe.getPage()); + + if (yieldSignal.isSet() || tryBuildPage()) { + return false; + } + } + return true; + } + + /** + * @return whether there are more positions on probe side + */ + private boolean advanceProbePosition(LookupSource lookupSource) + { + if (!probe.advanceNextPosition()) { + clearProbe(); + return false; + } + + // update join position + joinPosition = probe.getCurrentJoinPosition(lookupSource); + return true; + } + + private boolean tryBuildPage() + { + if (pageBuilder.isFull()) { + buildPage(); + return true; + } + return false; + } + + private void buildPage() + { + verify(outputPage == null); + verify(probe != null); + + if (pageBuilder.isEmpty()) { + return; + } + + outputPage = pageBuilder.build(probe); + pageBuilder.reset(); + } + + private void clearProbe() + { + // Before updating the probe flush the current page + buildPage(); + probe = null; + } + + @Override + public boolean supportsConsolidatedWrites() + { + return false; + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/LookupGroupJoinOperatorFactory.java b/presto-main/src/main/java/io/prestosql/operator/LookupGroupJoinOperatorFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..78e8a570c16066a07e9cd0edc59c58a1a566cff3 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/LookupGroupJoinOperatorFactory.java @@ -0,0 +1,405 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator; + +import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; +import io.prestosql.execution.Lifespan; +import io.prestosql.operator.GroupJoinProbe.GroupJoinProbeFactory; +import io.prestosql.operator.LookupJoinOperators.JoinType; +import io.prestosql.operator.aggregation.AccumulatorFactory; +import io.prestosql.operator.aggregation.partial.PartialAggregationController; +import io.prestosql.operator.groupjoin.ExecutionHelperFactory; +import io.prestosql.spi.plan.AggregationNode; +import io.prestosql.spi.plan.PlanNodeId; +import io.prestosql.spi.plan.Symbol; +import io.prestosql.spi.type.Type; +import io.prestosql.spiller.PartitioningSpillerFactory; +import io.prestosql.sql.gen.JoinCompiler; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.prestosql.SystemSessionProperties.isSpillToHdfsEnabled; +import static io.prestosql.operator.LookupJoinOperators.JoinType.INNER; +import static java.util.Objects.requireNonNull; + +public class LookupGroupJoinOperatorFactory + implements JoinOperatorFactory +{ + private final int operatorId; + private final PlanNodeId planNodeId; + private final List probeTypes; + private final List outputTypes; + private final List buildTypes; + private final LookupJoinOperators.JoinType joinType; + private final GroupJoinProbeFactory joinProbeFactory; + private final Optional outerOperatorFactoryResult; + private final JoinBridgeManager joinBridgeManager; + private ExecutionHelperFactory executionHelperFactory; + private final OptionalInt totalOperatorsCount; + private final HashGenerator probeHashGenerator; + private final boolean forked; + private final PartitioningSpillerFactory partitioningSpillerFactory; + private final List probeFinalOutputChannels; + private final List buildFinalOutputChannels; + private boolean closed; + private final GroupJoinAggregator aggrOnAggrfactory; + private final GroupJoinAggregator aggrfactory; + + private final PagesIndex.Factory pagesIndexFactory; + private final int expectedPositions; + + public static Builder builder() + { + return new Builder(); + } + + public LookupGroupJoinOperatorFactory( + int operatorId, + PlanNodeId planNodeId, + JoinBridgeManager lookupSourceFactoryManager, + List probeTypes, + List outputTypes, + List buildTypes, + JoinType joinType, + GroupJoinProbeFactory joinProbeFactory, + OptionalInt totalOperatorsCount, + List probeJoinChannels, + OptionalInt probeHashChannel, + PartitioningSpillerFactory partitioningSpillerFactory, + boolean forked, + GroupJoinAggregator aggrfactory, + GroupJoinAggregator aggrOnAggrfactory, + List probeFinalOutputChannels, + List buildFinalOutputChannels, + ExecutionHelperFactory executionHelperFactory, + int expectedPositions, + PagesIndex.Factory pagesIndexFactory) + { + this.forked = forked; + this.operatorId = operatorId; + this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); + this.probeTypes = ImmutableList.copyOf(requireNonNull(probeTypes, "probeTypes is null")); + this.outputTypes = ImmutableList.copyOf(requireNonNull(outputTypes, "outputTypes is null")); + this.buildTypes = ImmutableList.copyOf(requireNonNull(buildTypes, "buildTypes is null")); + this.joinType = requireNonNull(joinType, "joinType is null"); + this.joinProbeFactory = requireNonNull(joinProbeFactory, "joinProbeFactory is null"); + + this.joinBridgeManager = lookupSourceFactoryManager; + this.executionHelperFactory = requireNonNull(executionHelperFactory, "executionHelperFactory is null"); + joinBridgeManager.incrementProbeFactoryCount(); + checkArgument(joinType == INNER, "joinType is not INNER"); + this.outerOperatorFactoryResult = Optional.empty(); + this.totalOperatorsCount = requireNonNull(totalOperatorsCount, "totalOperatorsCount is null"); + + requireNonNull(probeHashChannel, "probeHashChannel is null"); + if (probeHashChannel.isPresent()) { + this.probeHashGenerator = new PrecomputedHashGenerator(probeHashChannel.getAsInt()); + } + else { + requireNonNull(probeJoinChannels, "probeJoinChannels is null"); + List hashTypes = probeJoinChannels.stream() + .map(probeTypes::get) + .collect(toImmutableList()); + this.probeHashGenerator = new InterpretedHashGenerator(hashTypes, probeJoinChannels); + } + this.partitioningSpillerFactory = partitioningSpillerFactory; + + this.aggrfactory = aggrfactory; + this.aggrOnAggrfactory = aggrOnAggrfactory; + this.probeFinalOutputChannels = probeFinalOutputChannels; + this.buildFinalOutputChannels = buildFinalOutputChannels; + this.expectedPositions = expectedPositions; + this.pagesIndexFactory = pagesIndexFactory; + } + + private LookupGroupJoinOperatorFactory(LookupGroupJoinOperatorFactory other) + { + requireNonNull(other, "other is null"); + checkArgument(!other.closed, "cannot duplicated closed OperatorFactory"); + + this.forked = true; + this.operatorId = other.operatorId; + this.planNodeId = other.planNodeId; + this.probeTypes = other.probeTypes; + this.outputTypes = other.outputTypes; + this.buildTypes = other.buildTypes; + this.joinType = other.joinType; + this.joinProbeFactory = other.joinProbeFactory; + this.joinBridgeManager = other.joinBridgeManager; + this.outerOperatorFactoryResult = other.outerOperatorFactoryResult; + this.totalOperatorsCount = other.totalOperatorsCount; + this.probeHashGenerator = other.probeHashGenerator; + this.partitioningSpillerFactory = other.partitioningSpillerFactory; + + this.aggrfactory = other.aggrfactory; + this.aggrOnAggrfactory = other.aggrOnAggrfactory; + + this.probeFinalOutputChannels = other.probeFinalOutputChannels; + this.buildFinalOutputChannels = other.buildFinalOutputChannels; + + this.expectedPositions = other.expectedPositions; + this.pagesIndexFactory = other.pagesIndexFactory; + this.closed = false; + this.joinBridgeManager.incrementProbeFactoryCount(); + } + + public int getOperatorId() + { + return operatorId; + } + + @Override + public Operator createOperator(DriverContext driverContext) + { + checkState(!closed, "Factory is already closed"); + LookupSourceFactory lookupSourceFactory = joinBridgeManager.getJoinBridge(driverContext.getLifespan()); + + OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LookupGroupJoinOperator.class.getSimpleName()); + lookupSourceFactory.setTaskContext(driverContext.getPipelineContext().getTaskContext()); + + joinBridgeManager.probeOperatorCreated(driverContext.getLifespan()); + return new LookupGroupJoinOperator( + operatorContext, + forked, + probeTypes, + outputTypes, + buildTypes, + joinType, + lookupSourceFactory, + joinProbeFactory, + () -> joinBridgeManager.probeOperatorClosed(driverContext.getLifespan()), + totalOperatorsCount, + probeHashGenerator, + partitioningSpillerFactory, + () -> joinBridgeManager.probeOperatorFinished(driverContext.getLifespan()), + isSpillToHdfsEnabled(driverContext.getPipelineContext().getTaskContext().getSession()), + aggrfactory, + aggrOnAggrfactory, + probeFinalOutputChannels, + buildFinalOutputChannels, + executionHelperFactory, + expectedPositions, + pagesIndexFactory); + } + + @Override + public void noMoreOperators() + { + checkState(!closed); + closed = true; + joinBridgeManager.probeOperatorFactoryClosedForAllLifespans(); + } + + @Override + public void noMoreOperators(Lifespan lifespan) + { + joinBridgeManager.probeOperatorFactoryClosed(lifespan); + } + + @Override + public OperatorFactory duplicate() + { + return new LookupGroupJoinOperatorFactory(this); + } + + @Override + public Optional createOuterOperatorFactory() + { + return outerOperatorFactoryResult; + } + + public LookupSourceFactory getLookupSourceFactory(Lifespan lifespan) + { + return joinBridgeManager.getJoinBridge(lifespan); + } + + public static class Builder + { + private GroupJoinAggregator aggrOnAggrfactory; + private GroupJoinAggregator aggrfactory; + private List probeFinalOutputSymbols; + private List probeFinalOutputChannels; + private List buildFinalOutputChannels; + + private int operatorId; + private PlanNodeId planNodeId; + private List probeTypes; + private List outputTypes; + private List buildTypes; + private LookupJoinOperators.JoinType joinType; + private GroupJoinProbeFactory joinProbeFactory; + private JoinBridgeManager joinBridgeManager; + private OptionalInt totalOperatorsCount; + private boolean forked; + private PartitioningSpillerFactory partitioningSpillerFactory; + private OptionalInt probeHashChannel; + private List probeJoinChannels; + private ExecutionHelperFactory executionHelperFactory; + + private PagesIndex.Factory pagesIndexFactory; + private int expectedPositions; + + public Builder() + { + } + + public Builder withExecutorHelperFactory(ExecutionHelperFactory executionHelperFactory) + { + this.executionHelperFactory = requireNonNull(executionHelperFactory, "executionHelperFactory is null"); + return this; + } + + public Builder withJoinInfo(int operatorId, + PlanNodeId planNodeId, + JoinBridgeManager lookupSourceFactoryManager, + List probeTypes, + List outputTypes, + List buildTypes, + JoinType joinType, + GroupJoinProbeFactory joinProbeFactory, + OptionalInt totalOperatorsCount, + List probeJoinChannels, + OptionalInt probeHashChannel, + PartitioningSpillerFactory partitioningSpillerFactory, + boolean forked) + { + this.operatorId = operatorId; + this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); + this.forked = forked; + this.probeTypes = ImmutableList.copyOf(requireNonNull(probeTypes, "probeTypes is null")); + this.outputTypes = ImmutableList.copyOf(requireNonNull(outputTypes, "outputTypes is null")); + this.buildTypes = ImmutableList.copyOf(requireNonNull(buildTypes, "buildTypes is null")); + this.joinType = requireNonNull(joinType, "joinType is null"); + this.joinProbeFactory = requireNonNull(joinProbeFactory, "joinProbeFactory is null"); + this.joinBridgeManager = lookupSourceFactoryManager; + checkArgument(joinType == INNER, "joinType is not INNER"); + this.totalOperatorsCount = requireNonNull(totalOperatorsCount, "totalOperatorsCount is null"); + requireNonNull(probeHashChannel, "probeHashChannel is null"); + this.probeHashChannel = probeHashChannel; + this.probeJoinChannels = probeJoinChannels; + this.partitioningSpillerFactory = partitioningSpillerFactory; + return this; + } + + public Builder withProbeOutputInfo(List probeFinalOutputSymbols, + List probeFinalOutputChannels) + { + this.probeFinalOutputChannels = ImmutableList.copyOf(probeFinalOutputChannels); + this.probeFinalOutputSymbols = ImmutableList.copyOf(probeFinalOutputSymbols); + return this; + } + + public Builder withBuildOutputInfo(List buildFinalOutputChannels) + { + this.buildFinalOutputChannels = ImmutableList.copyOf(buildFinalOutputChannels); + return this; + } + + public Builder withAggrOnAggrFactory(List groupByTypes, + List groupByChannels, + List globalAggregationGroupIds, + AggregationNode.Step step, + List accumulatorFactories, + Optional hashChannel, + Optional groupIdChannel, + int expectedGroups, + Optional maxPartialMemory, + JoinCompiler joinCompiler, + boolean useSystemMemory, + Optional partialAggregationController, + boolean produceDefaultOutput) + { + aggrOnAggrfactory = new GroupJoinAggregator(hashChannel, + groupIdChannel, + accumulatorFactories, + groupByTypes, + groupByChannels, + globalAggregationGroupIds, + step, + expectedGroups, + maxPartialMemory, + joinCompiler, + useSystemMemory, + partialAggregationController, + produceDefaultOutput); + return this; + } + + public Builder withAggrFactory(List groupByTypes, + List groupByChannels, + List globalAggregationGroupIds, + AggregationNode.Step step, + List accumulatorFactories, + Optional hashChannel, + Optional groupIdChannel, + int expectedGroups, + Optional maxPartialMemory, + JoinCompiler joinCompiler, + boolean useSystemMemory, + Optional partialAggregationController, + boolean produceDefaultOutput) + { + aggrfactory = new GroupJoinAggregator(hashChannel, + groupIdChannel, + accumulatorFactories, + groupByTypes, + groupByChannels, + globalAggregationGroupIds, + step, + expectedGroups, + maxPartialMemory, + joinCompiler, + useSystemMemory, + partialAggregationController, + produceDefaultOutput); + return this; + } + + public LookupGroupJoinOperatorFactory build() + { + requireNonNull(aggrfactory, "aggrfactory is null"); + requireNonNull(aggrOnAggrfactory, "aggrOnAggrfactory is null"); + requireNonNull(buildFinalOutputChannels, "buildFinalOutputChannels is null"); + requireNonNull(probeFinalOutputChannels, "buildFinalOutputChannels is null"); + requireNonNull(joinProbeFactory, "joinProbeFactory is null"); + return new LookupGroupJoinOperatorFactory( + operatorId, + planNodeId, + joinBridgeManager, + probeTypes, + outputTypes, + buildTypes, + joinType, + joinProbeFactory, + totalOperatorsCount, + probeJoinChannels, + probeHashChannel, + partitioningSpillerFactory, + forked, + aggrfactory, + aggrOnAggrfactory, + probeFinalOutputChannels, + buildFinalOutputChannels, + executionHelperFactory, + expectedPositions, + pagesIndexFactory); + } + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/LookupGroupJoinPageBuilder.java b/presto-main/src/main/java/io/prestosql/operator/LookupGroupJoinPageBuilder.java new file mode 100644 index 0000000000000000000000000000000000000000..809ac19f8c92aef344916754808349cc0d50ef80 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/LookupGroupJoinPageBuilder.java @@ -0,0 +1,167 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator; + +import com.google.common.collect.ImmutableList; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; +import io.prestosql.spi.Page; +import io.prestosql.spi.PageBuilder; +import io.prestosql.spi.snapshot.BlockEncodingSerdeProvider; +import io.prestosql.spi.snapshot.Restorable; +import io.prestosql.spi.type.Type; + +import java.util.List; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static io.prestosql.spi.block.PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; +import static java.util.Objects.requireNonNull; + +/** + * This page builder creates pages for group join after applying aggregator on probe and build side based on count. + */ +public class LookupGroupJoinPageBuilder + implements Restorable +{ + private final PageBuilder finalPageBuilder; + private final PageBuilder buildPageBuilderTmp; + private final List outputBuildChannels; + private final List outputProbeChannels; + private int estimatedProbeBlockBytes; + + public LookupGroupJoinPageBuilder(List outputTypes, List buildTypes, + List outputBuildChannels, List outputProbeChannels) + { + this.outputBuildChannels = requireNonNull(outputBuildChannels, "outputBuildChannels is null"); + this.outputProbeChannels = requireNonNull(outputProbeChannels, "outputProbeChannels is null"); + this.finalPageBuilder = new PageBuilder(ImmutableList.copyOf(requireNonNull(outputTypes, "outputTypes is null"))); + this.buildPageBuilderTmp = new PageBuilder(requireNonNull(buildTypes, "buildTypes is null")); + } + + public boolean isFull() + { + return estimatedProbeBlockBytes + finalPageBuilder.getSizeInBytes() >= DEFAULT_MAX_PAGE_SIZE_IN_BYTES || finalPageBuilder.isFull(); + } + + public boolean isEmpty() + { + return finalPageBuilder.isEmpty(); + } + + public void reset() + { + finalPageBuilder.reset(); + buildPageBuilderTmp.reset(); + estimatedProbeBlockBytes = 0; + } + + /** + * append the index for the probe and copy the row for the build + */ + public void appendRow(GroupJoinProbe probe, LookupSource lookupSource, long joinPosition) + { + // count is stored in last channel. + long buildCount = lookupSource.getCountForJoinPosition(joinPosition, lookupSource.getChannelCount() - 1); + long probeCount = probe.getCountProbeRecord(); + Page probePage = probe.getPage().getRegion(probe.getPosition(), 1); + buildPageBuilderTmp.declarePosition(); + lookupSource.appendTo(joinPosition, buildPageBuilderTmp, 0); + Page buildPage = buildPageBuilderTmp.build(); + + // probe side + Page probeFinalPage = null; + if (outputProbeChannels.size() != 0) { + probeFinalPage = processAggregationOnPage(probe.getProbeAggregationBuilder().getAggregationCount() == 0 ? 1 : buildCount, + probePage, + probe.getProbeAggregationBuilder()); + } + + // build side + Page buildFinalPage = null; + if (outputBuildChannels.size() != 0) { + buildFinalPage = processAggregationOnPage(probe.getBuildAggregationBuilder().getAggregationCount() == 0 ? 1 : probeCount, + buildPage, + probe.getBuildAggregationBuilder()); + } + + int probeChannelLength = outputProbeChannels.size(); + finalPageBuilder.declarePosition(); + for (int i = 0; i < probeChannelLength; i++) { + if (probeFinalPage.getBlock(outputProbeChannels.get(i)).isNull(0)) { + finalPageBuilder.getBlockBuilder(i).appendNull(); + continue; + } + probeFinalPage.getBlock(outputProbeChannels.get(i)).writePositionTo(0, finalPageBuilder.getBlockBuilder(i)); + } + + for (int i = 0; i < outputBuildChannels.size(); i++) { + if (buildFinalPage.getBlock(outputBuildChannels.get(i)).isNull(0)) { + finalPageBuilder.getBlockBuilder(i + probeChannelLength).appendNull(); + continue; + } + buildFinalPage.getBlock(outputBuildChannels.get(i)).writePositionTo(0, finalPageBuilder.getBlockBuilder(i + probeChannelLength)); + } + buildPageBuilderTmp.reset(); + } + + private Page processAggregationOnPage(long count, Page sourcePage, AggregationBuilder aggregationBuilder) + { + // TODO Vineet check on how to convert into future object and relate in normal code flow. + if (count == 1) { + return sourcePage; + } + Page finalPage; + for (int i = 0; i < count; i++) { + Work work = aggregationBuilder.processPage(sourcePage); + // Knowingly kept empty while loop + boolean process = work.process(); + while (!process) { + process = work.process(); + } + } + WorkProcessor pageWorkProcessor = aggregationBuilder.buildResult(); + boolean process = pageWorkProcessor.process(); + while (!process) { + process = pageWorkProcessor.process(); + } + aggregationBuilder.updateMemory(); + finalPage = pageWorkProcessor.getResult(); + return finalPage; + } + + public Page build(GroupJoinProbe probe) + { + return finalPageBuilder.build(); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("estimatedSize", estimatedProbeBlockBytes + finalPageBuilder.getSizeInBytes()) + .add("positionCount", finalPageBuilder.getPositionCount()) + .toString(); + } + + @Override + public Object capture(BlockEncodingSerdeProvider serdeProvider) + { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public void restore(Object state, BlockEncodingSerdeProvider serdeProvider) + { + throw new UnsupportedOperationException("Not supported"); + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/LookupJoinOperators.java b/presto-main/src/main/java/io/prestosql/operator/LookupJoinOperators.java index 09779bf15ddaff5eb1d318e0859d565cd69724f3..86a735fcd4341aa03c7645255fb72a0891829d48 100644 --- a/presto-main/src/main/java/io/prestosql/operator/LookupJoinOperators.java +++ b/presto-main/src/main/java/io/prestosql/operator/LookupJoinOperators.java @@ -14,6 +14,7 @@ package io.prestosql.operator; import io.prestosql.operator.JoinProbe.JoinProbeFactory; +import io.prestosql.operator.groupjoin.ExecutionHelperFactory; import io.prestosql.spi.plan.PlanNodeId; import io.prestosql.spi.type.Type; import io.prestosql.spiller.PartitioningSpillerFactory; @@ -99,4 +100,48 @@ public class LookupJoinOperators probeHashChannel, partitioningSpillerFactory); } + + public OperatorFactory groupInnerJoin( + int operatorId, + PlanNodeId planNodeId, + JoinBridgeManager lookupSourceFactoryManager, + List probeTypes, + List probeJoinChannels, + OptionalInt probeHashChannel, + OptionalInt probeCountChannel, + List probeOutputChannels, + OptionalInt totalOperatorsCount, + PartitioningSpillerFactory partitioningSpillerFactory, + boolean forked, + GroupJoinAggregator aggrfactory, + GroupJoinAggregator aggrOnAggrfactory, + List probeFinalOutputChannels, + List buildFinalOutputChannels, + List outputTypes, + ExecutionHelperFactory executionHelperFactory, + int expectedPositions, + PagesIndex.Factory pagesIndexFactory) + { + return new LookupGroupJoinOperatorFactory( + operatorId, + planNodeId, + lookupSourceFactoryManager, + probeTypes, + outputTypes, + lookupSourceFactoryManager.getBuildOutputTypes(), + JoinType.INNER, + new GroupJoinProbe.GroupJoinProbeFactory(probeOutputChannels.stream().mapToInt(i -> i).toArray(), probeJoinChannels, probeHashChannel, probeCountChannel), + totalOperatorsCount, + probeJoinChannels, + probeHashChannel, + partitioningSpillerFactory, + forked, + aggrfactory, + aggrOnAggrfactory, + probeFinalOutputChannels, + buildFinalOutputChannels, + executionHelperFactory, + expectedPositions, + pagesIndexFactory); + } } diff --git a/presto-main/src/main/java/io/prestosql/operator/LookupSource.java b/presto-main/src/main/java/io/prestosql/operator/LookupSource.java index 24399e242dcf9c452376c0f417469a3cf1337c3d..f974639e5fc3fd76c1f9ab308138888360b408ba 100644 --- a/presto-main/src/main/java/io/prestosql/operator/LookupSource.java +++ b/presto-main/src/main/java/io/prestosql/operator/LookupSource.java @@ -13,6 +13,7 @@ */ package io.prestosql.operator; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; @@ -38,6 +39,16 @@ public interface LookupSource long getNextJoinPosition(long currentJoinPosition, int probePosition, Page allProbeChannelsPage); + default long getCountForJoinPosition(long position, int channel) + { + throw new UnsupportedOperationException("Only supported for Group Join usage"); + } + + default AggregationBuilder getAggregationBuilder() + { + throw new UnsupportedOperationException("Only supported for Group Join usage"); + } + void appendTo(long position, PageBuilder pageBuilder, int outputChannelOffset); boolean isJoinPositionEligible(long currentJoinPosition, int probePosition, Page allProbeChannelsPage); diff --git a/presto-main/src/main/java/io/prestosql/operator/PagesHashStrategy.java b/presto-main/src/main/java/io/prestosql/operator/PagesHashStrategy.java index 0092e5630257c7380b4a3e6a37ebcfc65a56e64e..43c802787853a2a98bb076ffe7a1a28f34e6f62d 100644 --- a/presto-main/src/main/java/io/prestosql/operator/PagesHashStrategy.java +++ b/presto-main/src/main/java/io/prestosql/operator/PagesHashStrategy.java @@ -13,6 +13,7 @@ */ package io.prestosql.operator; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; @@ -110,4 +111,20 @@ public interface PagesHashStrategy * Checks if sort channel is null at the specified position */ boolean isSortChannelPositionNull(int blockIndex, int blockPosition); + + /** + * Gets the aggregation builder for build side. Used in GroupJoin + */ + default AggregationBuilder getAggregationBuilder() + { + throw new UnsupportedOperationException("Only supported for Group Join"); + } + + /** + * Gets the count (occurrence) of record from build side of the record. + */ + default long getCountForJoinPosition(int blockIndex, int blockPosition, int channel) + { + throw new UnsupportedOperationException("Only supported for Group Join"); + } } diff --git a/presto-main/src/main/java/io/prestosql/operator/PagesIndex.java b/presto-main/src/main/java/io/prestosql/operator/PagesIndex.java index a7cedd971318808930b7d81791c2e349fd184959..df96f4eeeb80bb143dd10b594d34c2f86458df36 100644 --- a/presto-main/src/main/java/io/prestosql/operator/PagesIndex.java +++ b/presto-main/src/main/java/io/prestosql/operator/PagesIndex.java @@ -25,6 +25,7 @@ import io.prestosql.Session; import io.prestosql.geospatial.Rectangle; import io.prestosql.metadata.Metadata; import io.prestosql.operator.SpatialIndexBuilderOperator.SpatialPredicate; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; import io.prestosql.spi.block.Block; @@ -445,6 +446,7 @@ public class PagesIndex joinChannels, hashChannel, Optional.empty(), + Optional.empty(), metadata); } @@ -482,6 +484,20 @@ public class PagesIndex Optional sortChannel, List searchFunctionFactories, Optional> outputChannels) + { + return createLookupSourceSupplier(session, joinChannels, hashChannel, filterFunctionFactory, sortChannel, searchFunctionFactories, outputChannels, Optional.empty(), Optional.empty()); + } + + public LookupSourceSupplier createLookupSourceSupplier( + Session session, + List joinChannels, + OptionalInt hashChannel, + Optional filterFunctionFactory, + Optional sortChannel, + List searchFunctionFactories, + Optional> outputChannels, + Optional countChannel, + Optional aggregationBuilder) { List> channelLists = ImmutableList.copyOf(this.channels); if (!joinChannels.isEmpty()) { @@ -490,7 +506,7 @@ public class PagesIndex // OUTER joins into NestedLoopsJoin and remove "type == INNER" condition in LocalExecutionPlanner.visitJoin() try { - LookupSourceSupplierFactory lookupSourceFactory = joinCompiler.compileLookupSourceFactory(types, joinChannels, sortChannel, outputChannels); + LookupSourceSupplierFactory lookupSourceFactory = joinCompiler.compileLookupSourceFactory(types, joinChannels, sortChannel, outputChannels, countChannel, aggregationBuilder); return lookupSourceFactory.createLookupSourceSupplier( session, valueAddresses, @@ -498,7 +514,8 @@ public class PagesIndex hashChannel, filterFunctionFactory, sortChannel, - searchFunctionFactories); + searchFunctionFactories, + aggregationBuilder); } catch (Exception e) { log.error(e, "Lookup source compile failed for types=%s error=%s", types, e); @@ -513,6 +530,7 @@ public class PagesIndex joinChannels, hashChannel, sortChannel, + aggregationBuilder, metadata); return new JoinHashSupplier( @@ -565,6 +583,42 @@ public class PagesIndex }; } + public class PageIterator

+ extends AbstractIterator + { + private int pageCounter; + private boolean isFinished; + + @Override + protected Page computeNext() + { + if (pageCounter == channels[0].size()) { + if (isFinished) { + return endOfData(); + } + else { + return null; + } + } + + Block[] blocks = Stream.of(channels) + .map(channel -> channel.get(pageCounter)) + .toArray(Block[]::new); + pageCounter++; + return new Page(blocks); + } + + public void setFinished(boolean isFinished) + { + this.isFinished = isFinished; + } + } + + public PageIterator getPagesIterator() + { + return new PageIterator<>(); + } + public Iterator getSortedPages() { return new AbstractIterator() diff --git a/presto-main/src/main/java/io/prestosql/operator/PartitionedLookupSource.java b/presto-main/src/main/java/io/prestosql/operator/PartitionedLookupSource.java index 1ce4e1eedaf362d404dc022de06419e187598b32..cac185fd8a65231c0384ccab1bfd48bde32d9fb5 100644 --- a/presto-main/src/main/java/io/prestosql/operator/PartitionedLookupSource.java +++ b/presto-main/src/main/java/io/prestosql/operator/PartitionedLookupSource.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.io.Closer; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.operator.exchange.LocalPartitionGenerator; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; @@ -264,6 +265,20 @@ public class PartitionedLookupSource return lookupSource.isJoinPositionEligible(joinPosition, probePosition, allProbeChannelsPage); } + @Override + public long getCountForJoinPosition(long partitionedJoinPosition, int channel) + { + int partition = decodePartition(partitionedJoinPosition); + long joinPosition = decodeJoinPosition(partitionedJoinPosition); + return lookupSources[partition].getCountForJoinPosition(joinPosition, channel); + } + + @Override + public AggregationBuilder getAggregationBuilder() + { + return lookupSources[0].getAggregationBuilder(); + } + @Override public void appendTo(long partitionedJoinPosition, PageBuilder pageBuilder, int outputChannelOffset) { diff --git a/presto-main/src/main/java/io/prestosql/operator/SimplePagesHashStrategy.java b/presto-main/src/main/java/io/prestosql/operator/SimplePagesHashStrategy.java index a88e0f27f14f42aa06c3f263cfa395a7ac2b7307..62bcc3fb29ca9eb67a47647dfb93f2316dbf8420 100644 --- a/presto-main/src/main/java/io/prestosql/operator/SimplePagesHashStrategy.java +++ b/presto-main/src/main/java/io/prestosql/operator/SimplePagesHashStrategy.java @@ -16,6 +16,7 @@ package io.prestosql.operator; import com.google.common.collect.ImmutableList; import io.prestosql.metadata.FunctionAndTypeManager; import io.prestosql.metadata.Metadata; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; import io.prestosql.spi.block.Block; @@ -46,6 +47,7 @@ public class SimplePagesHashStrategy private final List precomputedHashChannel; private final Optional sortChannel; private final List distinctFromMethodHandles; + private final AggregationBuilder aggregationBuilder; public SimplePagesHashStrategy( List types, @@ -54,6 +56,7 @@ public class SimplePagesHashStrategy List hashChannels, OptionalInt precomputedHashChannel, Optional sortChannel, + Optional aggregationBuilder, Metadata metadata) { this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); @@ -69,6 +72,8 @@ public class SimplePagesHashStrategy this.precomputedHashChannel = null; } this.sortChannel = requireNonNull(sortChannel, "sortChannel is null"); + requireNonNull(aggregationBuilder, "aggregationBuilder is null"); + this.aggregationBuilder = aggregationBuilder.orElse(null); requireNonNull(metadata, "metadata is null"); FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager(); requireNonNull(metadata, "functionManager is null"); @@ -95,6 +100,18 @@ public class SimplePagesHashStrategy .sum(); } + @Override + public long getCountForJoinPosition(int blockIndex, int blockPosition, int channel) + { + return BIGINT.getLong(channels.get(channel).get(blockIndex), blockPosition); + } + + @Override + public AggregationBuilder getAggregationBuilder() + { + return aggregationBuilder; + } + @Override public void appendTo(int blockIndex, int position, PageBuilder pageBuilder, int outputChannelOffset) { diff --git a/presto-main/src/main/java/io/prestosql/operator/UnSpilledGroupJoinProbe.java b/presto-main/src/main/java/io/prestosql/operator/UnSpilledGroupJoinProbe.java new file mode 100644 index 0000000000000000000000000000000000000000..e96d158155069ef93408fb80b6ddc01aaca15735 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/UnSpilledGroupJoinProbe.java @@ -0,0 +1,165 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.operator; + +import io.prestosql.operator.aggregation.builder.AggregationBuilder; +import io.prestosql.spi.Page; +import io.prestosql.spi.block.Block; + +import javax.annotation.Nullable; + +import java.util.Arrays; +import java.util.List; +import java.util.OptionalInt; +import java.util.stream.IntStream; + +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static java.util.Objects.requireNonNull; + +/** + * UnSpilledGroupJoinProbe + * + * @since 21-Feb-2023 + */ +public class UnSpilledGroupJoinProbe + extends GroupJoinProbe +{ + private final int[] probeOutputChannels; + private final Page page; + private final long[] joinPositionCache; + private int position = -1; + + public UnSpilledGroupJoinProbe(int[] probeOutputChannels, Page page, List probeJoinChannels, OptionalInt probeHashChannel, Page probePage, LookupSource lookupSource, @Nullable Block probeHashBlock, OptionalInt probeCountChannel, AggregationBuilder probeAggregationBuilder, AggregationBuilder buildAggregationBuilder) + { + super(probeOutputChannels, page, probeJoinChannels, probeHashChannel, probeCountChannel, probeAggregationBuilder, buildAggregationBuilder); + + this.probeOutputChannels = requireNonNull(probeOutputChannels, "probeOutputChannels is null"); + this.page = requireNonNull(page, "page is null"); + + joinPositionCache = fillCache(lookupSource, page, probeHashBlock, probePage); + } + + public int[] getOutputChannels() + { + return probeOutputChannels; + } + + public boolean advanceNextPosition() + { + verify(++position <= page.getPositionCount(), "already finished"); + return !isFinished(); + } + + public boolean isFinished() + { + return position == page.getPositionCount(); + } + + @Override + public long getCurrentJoinPosition(LookupSource lookupSource) + { + return joinPositionCache[position]; + } + + public int getPosition() + { + return position; + } + + public Page getPage() + { + return page; + } + + private static long[] fillCache( + LookupSource lookupSource, + Page page, + Block probeHashBlock, + Page probePage) + { + int positionCount = page.getPositionCount(); + List nullableBlocks = IntStream.range(0, probePage.getChannelCount()) + .mapToObj(i -> probePage.getBlock(i)) + .filter(Block::mayHaveNull) + .collect(toImmutableList()); + + long[] joinPositionsCache = new long[positionCount]; + if (!nullableBlocks.isEmpty()) { + Arrays.fill(joinPositionsCache, -1); + boolean[] isNull = new boolean[positionCount]; + int nonNullCount = getIsNull(nullableBlocks, positionCount, isNull); + if (nonNullCount < positionCount) { + // We only store positions that are not null + int[] positions = new int[nonNullCount]; + nonNullCount = 0; + for (int i = 0; i < positionCount; i++) { + if (!isNull[i]) { + positions[nonNullCount] = i; + } + // This way less code is in the if branch and CPU should be able to optimize branch prediction better + nonNullCount += isNull[i] ? 0 : 1; + } + if (probeHashBlock != null) { + long[] hashes = new long[positionCount]; + for (int i = 0; i < positionCount; i++) { + hashes[i] = BIGINT.getLong(probeHashBlock, i); + } + lookupSource.getJoinPosition(positions, probePage, page, hashes, joinPositionsCache); + } + else { + lookupSource.getJoinPosition(positions, probePage, page, joinPositionsCache); + } + return joinPositionsCache; + } // else fall back to non-null path + } + int[] positions = new int[positionCount]; + for (int i = 0; i < positionCount; i++) { + positions[i] = i; + } + if (probeHashBlock != null) { + long[] hashes = new long[positionCount]; + for (int i = 0; i < positionCount; i++) { + hashes[i] = BIGINT.getLong(probeHashBlock, i); + } + lookupSource.getJoinPosition(positions, probePage, page, hashes, joinPositionsCache); + } + else { + lookupSource.getJoinPosition(positions, probePage, page, joinPositionsCache); + } + + return joinPositionsCache; + } + + private static int getIsNull(List nullableBlocks, int positionCount, boolean[] isNull) + { + for (int i = 0; i < nullableBlocks.size() - 1; i++) { + Block block = nullableBlocks.get(i); + for (int jPosition = 0; jPosition < positionCount; jPosition++) { + isNull[jPosition] |= block.isNull(jPosition); + } + } + // Last block will also calculate `nonNullCount` + int nonNullCount = 0; + Block lastBlock = nullableBlocks.get(nullableBlocks.size() - 1); + for (int jPosition = 0; jPosition < positionCount; jPosition++) { + isNull[jPosition] |= lastBlock.isNull(jPosition); + nonNullCount += isNull[jPosition] ? 0 : 1; + } + + return nonNullCount; + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/AccumulatorCompiler.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/AccumulatorCompiler.java index 90e1404244ad47e8f1a1dbf1ab298211851d1ccb..1d8ee01aef8014782c627c0ba67d3927b8206c2c 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/AccumulatorCompiler.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/AccumulatorCompiler.java @@ -199,6 +199,7 @@ public class AccumulatorCompiler if (grouped) { generateGroupedEvaluateFinal(definition, stateFields, metadata.getOutputFunction(), callSiteBinder); + generateReset(definition, stateFields); } else { generateEvaluateFinal(definition, stateFields, metadata.getOutputFunction(), callSiteBinder); @@ -917,6 +918,30 @@ public class AccumulatorCompiler body.ret(); } + private static void generateReset(ClassDefinition definition, List stateFields) + { + Parameter groupId = arg("groupId", int.class); + MethodDefinition method = definition.declareMethod(a(PUBLIC), "reset", type(void.class), groupId); + + BytecodeBlock body = method.getBody(); + Variable thisVariable = method.getThis(); + + List states = new ArrayList<>(); + + for (FieldDefinition stateField : stateFields) { + BytecodeExpression state = thisVariable.getField(stateField); + body.append(state.invoke("setGroupId", void.class, groupId.cast(long.class))); + states.add(state); + } + for (FieldDefinition stateField : stateFields) { + BytecodeExpression state = thisVariable.getField(stateField); + body.append(state.invoke("reset", void.class)); + states.add(state); + } + states.forEach(body::append); + body.ret(); + } + private static void generateEvaluateFinal( ClassDefinition definition, List stateFields, diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/GenericAccumulatorFactory.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/GenericAccumulatorFactory.java index b6c8236c4bb65d1c0af672a0088a57fc1066ff06..2bc084577e8b37e93699e1669538b8427b1cc2d9 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/GenericAccumulatorFactory.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/GenericAccumulatorFactory.java @@ -446,6 +446,12 @@ public class GenericAccumulatorFactory accumulator.evaluateFinal(groupId, output); } + @Override + public void reset(int groupId) + { + accumulator.reset(groupId); + } + @Override public void prepareFinal() { @@ -651,6 +657,12 @@ public class GenericAccumulatorFactory accumulator.evaluateFinal(groupId, output); } + @Override + public void reset(int groupId) + { + accumulator.reset(groupId); + } + @Override public void prepareFinal() { diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/GroupedAccumulator.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/GroupedAccumulator.java index eda316ee875fdfe4b51ebcad9dcdda20bcaabb56..c832097c110a34d3ee1185869d593e5194a32015 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/GroupedAccumulator.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/GroupedAccumulator.java @@ -38,4 +38,6 @@ public interface GroupedAccumulator void evaluateFinal(int groupId, BlockBuilder output); void prepareFinal(); + + void reset(int groupId); } diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/arrayagg/GroupArrayAggregationState.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/arrayagg/GroupArrayAggregationState.java index ae0eb6b910458d8dc58893df23ec18bd63c7416f..d94f2477015f97de39a49545f9931d74433749cb 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/arrayagg/GroupArrayAggregationState.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/arrayagg/GroupArrayAggregationState.java @@ -38,6 +38,12 @@ public final class GroupArrayAggregationState appendAtChannel(VALUE_CHANNEL, block, position); } + @Override + public void reset() + { + ArrayAggregationState.super.reset(); + } + @Override protected final void accept(ArrayAggregationStateConsumer consumer, PageBuilder pageBuilder, int currentPosition) { diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/arrayagg/LegacyArrayAggregationGroupState.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/arrayagg/LegacyArrayAggregationGroupState.java index 2e7923773f249ef7fb7ef00a1878f4be24bddf88..586f9f35303f202963d9a6c66dc90bfc0cc48a3c 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/arrayagg/LegacyArrayAggregationGroupState.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/arrayagg/LegacyArrayAggregationGroupState.java @@ -90,4 +90,10 @@ public class LegacyArrayAggregationGroupState verify(blockBuilder.getPositionCount() != 0); return false; } + + @Override + public void reset() + { + ArrayAggregationState.super.reset(); + } } diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/AggregationBuilder.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/AggregationBuilder.java index ccce6690543833486d0297ef9a8e0096f8052c4b..3777dfe2d7cfa02d2bff33a98208a0181717f8b3 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/AggregationBuilder.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/AggregationBuilder.java @@ -45,4 +45,14 @@ public interface AggregationBuilder ListenableFuture startMemoryRevoke(); void finishMemoryRevoke(); + + default AggregationBuilder duplicate() + { + throw new UnsupportedOperationException("Only supported for Group Join flow"); + } + + default int getAggregationCount() + { + throw new UnsupportedOperationException("Only supported for Group Join flow"); + } } diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryAggregationBuilder.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryAggregationBuilder.java index 3678b7f651539a8485780ac880be74096567b778..eb7da4b5c4298dbe5653c3f605c8d0c0bdd4996b 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryAggregationBuilder.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryAggregationBuilder.java @@ -55,11 +55,12 @@ import static java.util.Objects.requireNonNull; public abstract class InMemoryAggregationBuilder implements AggregationBuilder, Restorable { - protected final GroupBy groupBy; protected final List aggregators; protected final boolean partial; protected final OptionalLong maxPartialMemory; protected final UpdateMemory updateMemory; + + protected GroupBy groupBy; protected boolean full; public InMemoryAggregationBuilder( @@ -324,6 +325,11 @@ public abstract class InMemoryAggregationBuilder } } + public void reset(int groupId) + { + aggregation.reset(groupId); + } + public void setOutputPartial() { step = AggregationNode.Step.partialOutput(step); @@ -390,6 +396,10 @@ public abstract class InMemoryAggregationBuilder throw new UnsupportedOperationException(); } + protected void resetGroupBy() + { + } + private static class InMemoryAggregationBuilderState implements Serializable { diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryHashAggregationBuilder.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryHashAggregationBuilder.java index ba0a6fbc290187c709438bc20f2aebba981a0bf8..a3611f7d5f9479f437198ffecfc830c30974e1b8 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryHashAggregationBuilder.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryHashAggregationBuilder.java @@ -179,7 +179,12 @@ public class InMemoryHashAggregationBuilder } } - return WorkProcessor.ProcessState.ofResult(pageBuilder.build()); + try { + return WorkProcessor.ProcessState.ofResult(pageBuilder.build()); + } + finally { + resetGroupBy(); + } }); } diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryHashAggregationBuilderWithReset.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryHashAggregationBuilderWithReset.java new file mode 100644 index 0000000000000000000000000000000000000000..cccad19b60896c63dfe008792f6d2b57b2011604 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/builder/InMemoryHashAggregationBuilderWithReset.java @@ -0,0 +1,122 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator.aggregation.builder; + +import com.google.common.primitives.Ints; +import io.airlift.units.DataSize; +import io.prestosql.operator.OperatorContext; +import io.prestosql.operator.UpdateMemory; +import io.prestosql.operator.aggregation.AccumulatorFactory; +import io.prestosql.spi.plan.AggregationNode; +import io.prestosql.spi.type.Type; +import io.prestosql.sql.gen.JoinCompiler; +import it.unimi.dsi.fastutil.ints.IntIterator; + +import java.util.List; +import java.util.Optional; + +import static io.prestosql.SystemSessionProperties.isDictionaryAggregationEnabled; +import static io.prestosql.operator.GroupByHash.createGroupByHash; + +public class InMemoryHashAggregationBuilderWithReset + extends InMemoryHashAggregationBuilder +{ + private final List accumulatorFactories; + private final AggregationNode.Step step; + private final int expectedGroups; + private final List groupByTypes; + private final List groupByChannels; + private final Optional hashChannel; + private final OperatorContext operatorContext; + private final Optional maxPartialMemory; + private final JoinCompiler joinCompiler; + + public InMemoryHashAggregationBuilderWithReset( + List accumulatorFactories, + AggregationNode.Step step, + int expectedGroups, + List groupByTypes, + List groupByChannels, + Optional hashChannel, + OperatorContext operatorContext, + Optional maxPartialMemory, + JoinCompiler joinCompiler, + UpdateMemory updateMemory) + { + super(accumulatorFactories, + AggregationNode.Step.partialInput(step), + expectedGroups, + groupByTypes, + groupByChannels, + hashChannel, + isDictionaryAggregationEnabled(operatorContext.getSession()), + maxPartialMemory, + Optional.empty(), + joinCompiler, + updateMemory, + AggregationNode.AggregationType.HASH); + this.accumulatorFactories = accumulatorFactories; + this.step = AggregationNode.Step.partialInput(step); + this.expectedGroups = expectedGroups; + this.groupByTypes = groupByTypes; + this.groupByChannels = groupByChannels; + this.hashChannel = hashChannel; + this.operatorContext = operatorContext; + this.maxPartialMemory = maxPartialMemory; + this.joinCompiler = joinCompiler; + } + + @Override + protected void resetGroupBy() + { + IntIterator intIterator = consecutiveGroupIds(); + while (intIterator.hasNext()) { + int groupId = intIterator.nextInt(); + for (Aggregator aggregator : aggregators) { + aggregator.reset(groupId); + } + } + + this.groupBy = createGroupByHash( + groupByTypes, + Ints.toArray(groupByChannels), + hashChannel, + expectedGroups, + isDictionaryAggregationEnabled(operatorContext.getSession()), + joinCompiler, + updateMemory); + } + + @Override + public int getAggregationCount() + { + return accumulatorFactories.size(); + } + + @Override + public AggregationBuilder duplicate() + { + return new InMemoryHashAggregationBuilderWithReset( + accumulatorFactories, + step, + expectedGroups, + groupByTypes, + groupByChannels, + hashChannel, + operatorContext, + maxPartialMemory, + joinCompiler, + updateMemory); + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/multimapagg/GroupedMultimapAggregationState.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/multimapagg/GroupedMultimapAggregationState.java index a70639840fc3e4adb3611a2a2025bb9fdb30331f..4dd51feafea836a8a978938b4d260ba0ef658a04 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/multimapagg/GroupedMultimapAggregationState.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/multimapagg/GroupedMultimapAggregationState.java @@ -40,6 +40,12 @@ public final class GroupedMultimapAggregationState appendAtChannel(KEY_CHANNEL, keyBlock, position); } + @Override + public void reset() + { + MultimapAggregationState.super.reset(); + } + @Override protected final void accept(MultimapAggregationStateConsumer consumer, PageBuilder pageBuilder, int currentPosition) { diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/multimapagg/LegacyGroupedMultimapAggregationState.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/multimapagg/LegacyGroupedMultimapAggregationState.java index bd966d980d5e96a3eaabac3ed8514893076238c5..ce127c1a5958f7a97a30b9b8e4b0133d3e243446 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/multimapagg/LegacyGroupedMultimapAggregationState.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/multimapagg/LegacyGroupedMultimapAggregationState.java @@ -91,6 +91,12 @@ public class LegacyGroupedMultimapAggregationState return keyBlockBuilders.get(getGroupId()) == null; } + @Override + public void reset() + { + MultimapAggregationState.super.reset(); + } + @Override public long getEstimatedSize() { diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/state/AbstractGroupedAccumulatorState.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/state/AbstractGroupedAccumulatorState.java index cb5aeefaa0cc65bbd9f12905431826643aa16602..f4c9fcf1e8a4ee2b7f5e542f1bf67efb787a6cb6 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/state/AbstractGroupedAccumulatorState.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/state/AbstractGroupedAccumulatorState.java @@ -33,6 +33,10 @@ public abstract class AbstractGroupedAccumulatorState return groupId; } + protected void reset() + { + } + @Override public Object capture(BlockEncodingSerdeProvider serdeProvider) { diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/state/LongDecimalWithOverflowAndLongStateFactory.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/state/LongDecimalWithOverflowAndLongStateFactory.java index fad38fc25b006d978d733ccd7244321f968e3e7e..705f4832b7f0989324658b5a3eedc491a8ac9efc 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/state/LongDecimalWithOverflowAndLongStateFactory.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/state/LongDecimalWithOverflowAndLongStateFactory.java @@ -79,6 +79,13 @@ public class LongDecimalWithOverflowAndLongStateFactory longs.add(getGroupId(), value); } + @Override + public void reset() + { + super.reset(); + longs.reset(getGroupId()); + } + @Override public long getEstimatedSize() { diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/state/LongDecimalWithOverflowStateFactory.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/state/LongDecimalWithOverflowStateFactory.java index c3a4d8b9d671e612b8ef3058b354459c36801bde..98cf816dbaaafe434abb5756c34f30dc24062278 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/state/LongDecimalWithOverflowStateFactory.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/state/LongDecimalWithOverflowStateFactory.java @@ -90,6 +90,19 @@ public class LongDecimalWithOverflowStateFactory isNotNull.set(getGroupId(), true); } + @Override + public void reset() + { + isNotNull.reset(getGroupId()); + long[] decimalArray = getDecimalArray(); + int decimalArrayOffset = getDecimalArrayOffset(); + decimalArray[decimalArrayOffset] = 0; + decimalArray[decimalArrayOffset + 1] = 0; + if (overflows != null) { + overflows.reset(getGroupId()); + } + } + @Override public long[] getDecimalArray() { diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/state/StateCompiler.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/state/StateCompiler.java index 06e50cdd3084b0e589152da7cb71d64fdebac9ea..2eb6c89b55f0ea147bc256f21e0309ce0dbfe36a 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/state/StateCompiler.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/state/StateCompiler.java @@ -517,14 +517,17 @@ public class StateCompiler // Create ensureCapacity MethodDefinition ensureCapacity = definition.declareMethod(a(PUBLIC), "ensureCapacity", type(void.class), arg("size", long.class)); + MethodDefinition reset = definition.declareMethod(a(PUBLIC), "reset", type(void.class)); + // Generate fields, constructor, and ensureCapacity List fieldDefinitions = new ArrayList<>(); for (StateField field : fields) { - fieldDefinitions.add(generateGroupedField(definition, constructor, ensureCapacity, field)); + fieldDefinitions.add(generateGroupedField(definition, constructor, ensureCapacity, field, reset)); } constructor.getBody().ret(); ensureCapacity.getBody().ret(); + reset.getBody().ret(); // Generate getEstimatedSize MethodDefinition getEstimatedSize = definition.declareMethod(a(PUBLIC), "getEstimatedSize", type(long.class)); @@ -592,7 +595,7 @@ public class StateCompiler .append(constructor.getThis().setField(field, stateField.initialValueExpression())); } - private static FieldDefinition generateGroupedField(ClassDefinition definition, MethodDefinition constructor, MethodDefinition ensureCapacity, StateField stateField) + private static FieldDefinition generateGroupedField(ClassDefinition definition, MethodDefinition constructor, MethodDefinition ensureCapacity, StateField stateField, MethodDefinition reset) { Class bigArrayType = getBigArrayType(stateField.getType()); FieldDefinition field = definition.declareField(a(PRIVATE), UPPER_CAMEL.to(LOWER_CAMEL, stateField.getName()) + "Values", bigArrayType); @@ -621,6 +624,9 @@ public class StateCompiler ensureCapacity.getBody() .append(ensureCapacity.getThis().getField(field).invoke("ensureCapacity", void.class, ensureCapacityScope.getVariable("size"))); + reset.getBody() + .append(reset.getThis().getField(field).invoke("reset", void.class, reset.getThis().invoke("getGroupId", long.class))); + // Initialize field in constructor constructor.getBody() .append(constructor.getThis().setField(field, newInstance(field.getType(), stateField.initialValueExpression()))); diff --git a/presto-main/src/main/java/io/prestosql/operator/groupjoin/ExecutionHelper.java b/presto-main/src/main/java/io/prestosql/operator/groupjoin/ExecutionHelper.java new file mode 100644 index 0000000000000000000000000000000000000000..6e83faa25895148d53f0336a7abbf5514206ef0e --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/groupjoin/ExecutionHelper.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator.groupjoin; + +import com.google.common.util.concurrent.ListenableFuture; + +public interface ExecutionHelper +{ + ListenableFuture submitWork(Runnable work); +} diff --git a/presto-main/src/main/java/io/prestosql/operator/groupjoin/ExecutionHelperFactory.java b/presto-main/src/main/java/io/prestosql/operator/groupjoin/ExecutionHelperFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..ea295da780b0280da8b6240c39939bfd5ff8d607 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/groupjoin/ExecutionHelperFactory.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator.groupjoin; + +public interface ExecutionHelperFactory +{ + ExecutionHelper create(); +} diff --git a/presto-main/src/main/java/io/prestosql/operator/groupjoin/GeneralExecutionHelper.java b/presto-main/src/main/java/io/prestosql/operator/groupjoin/GeneralExecutionHelper.java new file mode 100644 index 0000000000000000000000000000000000000000..a5994c1e3f91d0c702bc7bcd45b7e76ea8fb4ce1 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/groupjoin/GeneralExecutionHelper.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator.groupjoin; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; + +public class GeneralExecutionHelper + implements ExecutionHelper +{ + private final ListeningExecutorService executor; + + public GeneralExecutionHelper(ListeningExecutorService executor) + { + this.executor = executor; + } + + @Override + public ListenableFuture submitWork(Runnable work) + { + return executor.submit(work); + } +} diff --git a/presto-main/src/main/java/io/prestosql/operator/groupjoin/GeneralExecutionHelperFactory.java b/presto-main/src/main/java/io/prestosql/operator/groupjoin/GeneralExecutionHelperFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..7289358e92074ed3e8c523f79dbaa60b6559014c --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/operator/groupjoin/GeneralExecutionHelperFactory.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.operator.groupjoin; + +import com.google.common.util.concurrent.ListeningExecutorService; +import io.prestosql.sql.analyzer.FeaturesConfig; + +import javax.annotation.PreDestroy; +import javax.inject.Inject; + +import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; +import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.Executors.newFixedThreadPool; + +public class GeneralExecutionHelperFactory + implements ExecutionHelperFactory +{ + private final ListeningExecutorService executor; + + @Inject + public GeneralExecutionHelperFactory(FeaturesConfig featuresConfig) + { + executor = listeningDecorator(newFixedThreadPool( + requireNonNull(featuresConfig, "featuresConfig is null").getGroupJoinThreads(), + daemonThreadsNamed("group-join-executor-%s"))); + } + + @Override + public ExecutionHelper create() + { + return new GeneralExecutionHelper(executor); + } + + @PreDestroy + public void destroy() + { + executor.shutdownNow(); + } +} diff --git a/presto-main/src/main/java/io/prestosql/server/ServerMainModule.java b/presto-main/src/main/java/io/prestosql/server/ServerMainModule.java index 70365a42b4d577d09b0c5c0686fb886672d2aa14..fdb124de2cb5e43d16b037c47dd88c0f21856f26 100644 --- a/presto-main/src/main/java/io/prestosql/server/ServerMainModule.java +++ b/presto-main/src/main/java/io/prestosql/server/ServerMainModule.java @@ -113,6 +113,8 @@ import io.prestosql.operator.ForExchange; import io.prestosql.operator.LookupJoinOperators; import io.prestosql.operator.OperatorStats; import io.prestosql.operator.PagesIndex; +import io.prestosql.operator.groupjoin.ExecutionHelperFactory; +import io.prestosql.operator.groupjoin.GeneralExecutionHelperFactory; import io.prestosql.operator.index.IndexJoinLookupStats; import io.prestosql.security.PasswordSecurityConfig; import io.prestosql.seedstore.SeedStoreManager; @@ -580,6 +582,7 @@ public class ServerMainModule binder.bind(SpillerFactory.class).to(GenericSpillerFactory.class).in(Scopes.SINGLETON); binder.bind(SingleStreamSpillerFactory.class).to(FileSingleStreamSpillerFactory.class).in(Scopes.SINGLETON); binder.bind(PartitioningSpillerFactory.class).to(GenericPartitioningSpillerFactory.class).in(Scopes.SINGLETON); + binder.bind(ExecutionHelperFactory.class).to(GeneralExecutionHelperFactory.class).in(Scopes.SINGLETON); binder.bind(SpillerStats.class).in(Scopes.SINGLETON); newExporter(binder).export(SpillerFactory.class).withGeneratedName(); binder.bind(LocalSpillManager.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/io/prestosql/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/io/prestosql/sql/analyzer/FeaturesConfig.java index 1b965093f80b5a55e68779ebf5dda31eb787747e..03c910b4097f506ed02df7ccae3152b76565637e 100644 --- a/presto-main/src/main/java/io/prestosql/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/io/prestosql/sql/analyzer/FeaturesConfig.java @@ -198,6 +198,7 @@ public class FeaturesConfig private DataSize cteMaterializationThresholdSize = new DataSize(128, MEGABYTE); private long joinPartitionedBuildMinRowCount = 1_000_000L; + private int groupJoinThreads = 4; @Config("optimizer.transform-self-join-to-window") public FeaturesConfig setTransformSelfJoinToWindow(boolean value) @@ -1673,4 +1674,17 @@ public class FeaturesConfig this.cteMaterializationThresholdSize = cteMaterializationThresholdSize; return this; } + + @Min(1) + public int getGroupJoinThreads() + { + return groupJoinThreads; + } + + @Config("experimental.group-join-threads") + public FeaturesConfig setGroupJoinThreads(int groupJoinThreads) + { + this.groupJoinThreads = groupJoinThreads; + return this; + } } diff --git a/presto-main/src/main/java/io/prestosql/sql/gen/JoinCompiler.java b/presto-main/src/main/java/io/prestosql/sql/gen/JoinCompiler.java index 5b952caf9361699a819350120b37402d708d07ee..692b3e5eda351189b36ddab1296ce3eacc422fbf 100644 --- a/presto-main/src/main/java/io/prestosql/sql/gen/JoinCompiler.java +++ b/presto-main/src/main/java/io/prestosql/sql/gen/JoinCompiler.java @@ -42,6 +42,7 @@ import io.prestosql.operator.JoinHash; import io.prestosql.operator.JoinHashSupplier; import io.prestosql.operator.LookupSourceSupplier; import io.prestosql.operator.PagesHashStrategy; +import io.prestosql.operator.aggregation.builder.AggregationBuilder; import io.prestosql.spi.Page; import io.prestosql.spi.PageBuilder; import io.prestosql.spi.block.Block; @@ -100,18 +101,18 @@ public class JoinCompiler .recordStats() .maximumSize(1000) .build(CacheLoader.from(key -> - internalCompileLookupSourceFactory(key.getTypes(), key.getOutputChannels(), key.getJoinChannels(), key.getSortChannel()))); + internalCompileLookupSourceFactory(key.getTypes(), key.getOutputChannels(), key.getJoinChannels(), key.getSortChannel(), key.getCountChannel(), key.getAggregationBuilder()))); private final LoadingCache> hashStrategies = CacheBuilder.newBuilder() .recordStats() .maximumSize(1000) .build(CacheLoader.from(key -> - internalCompileHashStrategy(key.getTypes(), key.getOutputChannels(), key.getJoinChannels(), key.getSortChannel()))); + internalCompileHashStrategy(key.getTypes(), key.getOutputChannels(), key.getJoinChannels(), key.getSortChannel(), key.getCountChannel(), key.getAggregationBuilder()))); private final boolean enableSingleChannelBigintLookupSource; public LookupSourceSupplierFactory compileLookupSourceFactory(List types, List joinChannels, Optional sortChannel) { - return compileLookupSourceFactory(types, joinChannels, sortChannel, Optional.empty()); + return compileLookupSourceFactory(types, joinChannels, sortChannel, Optional.empty(), Optional.empty(), Optional.empty()); } @Inject @@ -141,13 +142,16 @@ public class JoinCompiler return new CacheStatsMBean(hashStrategies); } - public LookupSourceSupplierFactory compileLookupSourceFactory(List types, List joinChannels, Optional sortChannel, Optional> outputChannels) + public LookupSourceSupplierFactory compileLookupSourceFactory(List types, List joinChannels, Optional sortChannel, Optional> outputChannels, + Optional countChannel, Optional aggregationBuilder) { return lookupSourceFactories.getUnchecked(new CacheKey( types, outputChannels.orElse(rangeList(types.size())), joinChannels, - sortChannel)); + sortChannel, + countChannel, + aggregationBuilder)); } public PagesHashStrategyFactory compilePagesHashStrategyFactory(List types, List joinChannels) @@ -165,6 +169,8 @@ public class JoinCompiler types, outputChannels.orElse(rangeList(types.size())), joinChannels, + Optional.empty(), + Optional.empty(), Optional.empty()))); } @@ -175,9 +181,10 @@ public class JoinCompiler .collect(toImmutableList()); } - private LookupSourceSupplierFactory internalCompileLookupSourceFactory(List types, List outputChannels, List joinChannels, Optional sortChannel) + private LookupSourceSupplierFactory internalCompileLookupSourceFactory(List types, List outputChannels, List joinChannels, Optional sortChannel, + Optional countChannel, Optional aggregationBuilder) { - Class pagesHashStrategyClass = internalCompileHashStrategy(types, outputChannels, joinChannels, sortChannel); + Class pagesHashStrategyClass = internalCompileHashStrategy(types, outputChannels, joinChannels, sortChannel, countChannel, aggregationBuilder); OptionalInt singleBigintJoinChannel = OptionalInt.empty(); if (enableSingleChannelBigintLookupSource @@ -213,7 +220,8 @@ public class JoinCompiler return instanceSize; } - private Class internalCompileHashStrategy(List types, List outputChannels, List joinChannels, Optional sortChannel) + private Class internalCompileHashStrategy(List types, List outputChannels, List joinChannels, Optional sortChannel, + Optional countChannel, Optional aggregationBuilder) { CallSiteBinder callSiteBinder = new CallSiteBinder(); @@ -238,10 +246,13 @@ public class JoinCompiler joinChannelFields.add(channelField); } FieldDefinition hashChannelField = classDefinition.declareField(a(PRIVATE, FINAL), "hashChannel", type(List.class, Block.class)); + FieldDefinition aggregationBuilderField = classDefinition.declareField(a(PRIVATE, FINAL), "aggregationBuilder", type(AggregationBuilder.class)); - generateConstructor(classDefinition, joinChannels, sizeField, instanceSizeField, channelFields, joinChannelFields, hashChannelField); + generateConstructor(classDefinition, joinChannels, sizeField, instanceSizeField, channelFields, joinChannelFields, hashChannelField, aggregationBuilderField); generateGetChannelCountMethod(classDefinition, outputChannels.size()); + generateGetterAggregationBuilder(classDefinition, aggregationBuilderField); generateGetSizeInBytesMethod(classDefinition, sizeField); + generateGetCountForJoinPosition(classDefinition, callSiteBinder, channelFields, countChannel); generateAppendToMethod(classDefinition, callSiteBinder, types, outputChannels, channelFields); generateHashPositionMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields, hashChannelField); generateHashRowMethod(classDefinition, callSiteBinder, joinChannelTypes); @@ -259,17 +270,54 @@ public class JoinCompiler return defineClass(classDefinition, PagesHashStrategy.class, callSiteBinder.getBindings(), getClass().getClassLoader()); } + private void generateGetCountForJoinPosition(ClassDefinition classDefinition, CallSiteBinder callSiteBinder, List channelFields, Optional countChannel) + { + Parameter blockIndex = arg("blockIndex", int.class); + Parameter blockPosition = arg("blockPosition", int.class); + Parameter channel = arg("channel", int.class); + MethodDefinition getCountForJoinPosition = classDefinition.declareMethod( + a(PUBLIC), + "getCountForJoinPosition", + type(long.class), + blockIndex, + blockPosition, + channel); + + if (!countChannel.isPresent()) { + getCountForJoinPosition.getBody() + .append(newInstance(UnsupportedOperationException.class)) + .throwObject(); + return; + } + Variable thisVariable = getCountForJoinPosition.getThis(); + BytecodeBlock body = getCountForJoinPosition.getBody(); + BytecodeExpression block = thisVariable + .getField(channelFields.get(countChannel.get())) + .invoke("get", Object.class, blockIndex) + .cast(Block.class); + BytecodeExpression bigintType = constantType(callSiteBinder, BIGINT); + BytecodeExpression getLong = bigintType.invoke( + "getLong", + long.class, + block, + blockPosition) + .ret(); + body.append(getLong); + } + private static void generateConstructor(ClassDefinition classDefinition, List joinChannels, FieldDefinition sizeField, FieldDefinition instanceSizeField, List channelFields, List joinChannelFields, - FieldDefinition hashChannelField) + FieldDefinition hashChannelField, + FieldDefinition aggregationBuilderField) { Parameter channels = arg("channels", type(List.class, type(List.class, Block.class))); Parameter hashChannel = arg("hashChannel", type(OptionalInt.class)); - MethodDefinition constructorDefinition = classDefinition.declareConstructor(a(PUBLIC), channels, hashChannel); + Parameter aggregationBuilder = arg("aggregationBuilder", type(Optional.class, AggregationBuilder.class)); + MethodDefinition constructorDefinition = classDefinition.declareConstructor(a(PUBLIC), channels, hashChannel, aggregationBuilder); Variable thisVariable = constructorDefinition.getThis(); Variable blockIndex = constructorDefinition.getScope().declareVariable(int.class, "blockIndex"); @@ -331,6 +379,16 @@ public class JoinCompiler .ifFalse(thisVariable.setField( hashChannelField, constantNull(hashChannelField.getType())))); + + constructor.comment("Set aggregationBuilder"); + constructor.append(new IfStatement() + .condition(aggregationBuilder.invoke("isPresent", boolean.class)) + .ifTrue(thisVariable.setField( + aggregationBuilderField, + aggregationBuilder.invoke("get", Object.class).cast(AggregationBuilder.class))) + .ifFalse(thisVariable.setField( + aggregationBuilderField, + constantNull(aggregationBuilderField.getType())))); constructor.ret(); } @@ -345,6 +403,19 @@ public class JoinCompiler .retInt(); } + private static void generateGetterAggregationBuilder(ClassDefinition classDefinition, FieldDefinition aggregationBuilder) + { + MethodDefinition getAggregationBuilder = classDefinition.declareMethod( + a(PUBLIC), + "getAggregationBuilder", + type(AggregationBuilder.class)); + Variable thisVariable = getAggregationBuilder.getThis(); + getAggregationBuilder + .getBody() + .append(thisVariable.getField(aggregationBuilder)) + .ret(AggregationBuilder.class); + } + private static void generateGetSizeInBytesMethod(ClassDefinition classDefinition, FieldDefinition sizeField) { MethodDefinition getSizeInBytesMethod = classDefinition.declareMethod(a(PUBLIC), "getSizeInBytes", type(long.class)); @@ -928,9 +999,10 @@ public class JoinCompiler OptionalInt hashChannel, Optional filterFunctionFactory, Optional sortChannel, - List searchFunctionFactories) + List searchFunctionFactories, + Optional aggregationBuilder) { - PagesHashStrategy pagesHashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel); + PagesHashStrategy pagesHashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel, aggregationBuilder); try { return constructor.newInstance(session, pagesHashStrategy, addresses, channels, filterFunctionFactory, sortChannel, searchFunctionFactories, singleBigintJoinChannel); } @@ -947,7 +1019,7 @@ public class JoinCompiler public PagesHashStrategyFactory(Class pagesHashStrategyClass) { try { - constructor = pagesHashStrategyClass.getConstructor(List.class, OptionalInt.class); + constructor = pagesHashStrategyClass.getConstructor(List.class, OptionalInt.class, Optional.class); } catch (NoSuchMethodException e) { throw new RuntimeException(e); @@ -957,7 +1029,17 @@ public class JoinCompiler public PagesHashStrategy createPagesHashStrategy(List> channels, OptionalInt hashChannel) { try { - return constructor.newInstance(channels, hashChannel); + return constructor.newInstance(channels, hashChannel, Optional.empty()); + } + catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + + public PagesHashStrategy createPagesHashStrategy(List> channels, OptionalInt hashChannel, Optional aggregationBuilder) + { + try { + return constructor.newInstance(channels, hashChannel, aggregationBuilder); } catch (ReflectiveOperationException e) { throw new RuntimeException(e); @@ -971,13 +1053,17 @@ public class JoinCompiler private final List outputChannels; private final List joinChannels; private final Optional sortChannel; + private final Optional countChannel; + private final Optional aggregationBuilder; - private CacheKey(List types, List outputChannels, List joinChannels, Optional sortChannel) + private CacheKey(List types, List outputChannels, List joinChannels, Optional sortChannel, Optional countChannel, Optional aggregationBuilder) { this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); this.outputChannels = ImmutableList.copyOf(requireNonNull(outputChannels, "outputChannels is null")); this.joinChannels = ImmutableList.copyOf(requireNonNull(joinChannels, "joinChannels is null")); this.sortChannel = requireNonNull(sortChannel, "sortChannel is null"); + this.countChannel = requireNonNull(countChannel, "countChannel is null"); + this.aggregationBuilder = requireNonNull(aggregationBuilder, "aggregationBuilder is null"); } private List getTypes() @@ -1000,10 +1086,20 @@ public class JoinCompiler return sortChannel; } + private Optional getCountChannel() + { + return countChannel; + } + + private Optional getAggregationBuilder() + { + return aggregationBuilder; + } + @Override public int hashCode() { - return Objects.hash(types, outputChannels, joinChannels, sortChannel); + return Objects.hash(types, outputChannels, joinChannels, sortChannel, countChannel, aggregationBuilder); } @Override @@ -1019,7 +1115,9 @@ public class JoinCompiler return Objects.equals(this.types, other.types) && Objects.equals(this.outputChannels, other.outputChannels) && Objects.equals(this.joinChannels, other.joinChannels) && - Objects.equals(this.sortChannel, other.sortChannel); + Objects.equals(this.sortChannel, other.sortChannel) && + Objects.equals(this.countChannel, other.countChannel) && + Objects.equals(this.aggregationBuilder, other.aggregationBuilder); } } } diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/DistributedExecutionPlanner.java b/presto-main/src/main/java/io/prestosql/sql/planner/DistributedExecutionPlanner.java index 798a80acad5b75cba7f92eb155dc4994dabb2816..db688474fadfa505928976a30c4d00300cbc3788 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/DistributedExecutionPlanner.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/DistributedExecutionPlanner.java @@ -40,6 +40,7 @@ import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; @@ -391,6 +392,17 @@ public class DistributedExecutionPlanner .build(); } + @Override + public Map visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + Map leftSplits = node.getLeft().accept(this, context); + Map rightSplits = node.getRight().accept(this, context); + return ImmutableMap.builder() + .putAll(leftSplits) + .putAll(rightSplits) + .build(); + } + @Override public Map visitSemiJoin(SemiJoinNode node, Void context) { @@ -770,6 +782,15 @@ public class DistributedExecutionPlanner return ret; } + @Override + public Map visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + pendingJoins++; + Map ret = super.visitJoinOnAggregation(node, context); + sourceStack.pop(); + return ret; + } + @Override public Map visitSemiJoin(SemiJoinNode node, Void context) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/ExpressionExtractor.java b/presto-main/src/main/java/io/prestosql/sql/planner/ExpressionExtractor.java index fd1be43d6d9bb36ad41264b02ea824762a387faf..d9664f709aaefc3054803806ba7fde82ab6d0319 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/ExpressionExtractor.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/ExpressionExtractor.java @@ -19,6 +19,7 @@ import io.prestosql.spi.plan.AggregationNode.Aggregation; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupReference; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.ProjectNode; import io.prestosql.spi.plan.ValuesNode; @@ -121,6 +122,13 @@ public final class ExpressionExtractor return super.visitJoin(node, context); } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Consumer context) + { + node.getFilter().ifPresent(context); + return super.visitJoinOnAggregation(node, context); + } + @Override public Void visitValues(ValuesNode node, Consumer context) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/LocalDynamicFilter.java b/presto-main/src/main/java/io/prestosql/sql/planner/LocalDynamicFilter.java index bb4dab8dd19174bbbdb34dfa05dfd1479e636fe7..0e9f026153472b949ca557ece419acceec7990fb 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/LocalDynamicFilter.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/LocalDynamicFilter.java @@ -28,6 +28,7 @@ import io.prestosql.spi.dynamicfilter.BloomFilterDynamicFilter; import io.prestosql.spi.dynamicfilter.DynamicFilter; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.Symbol; import io.prestosql.spi.predicate.TupleDomain; @@ -163,6 +164,52 @@ public class LocalDynamicFilter return Optional.of(new LocalDynamicFilter(localProbeSymbols, localBuildChannels, partitionCount, localType, session, taskId, stateStoreProvider)); } + public static Optional create(JoinOnAggregationNode planNode, int partitionCount, Session session, TaskId taskId, StateStoreProvider stateStoreProvider) + { + Set joinDynamicFilters = planNode.getDynamicFilters().keySet(); + // Mapping from probe-side dynamic filters' IDs to their matching probe symbols. + Multimap localProbeSymbols = MultimapBuilder.treeKeys().arrayListValues().build(); + PlanNode buildNode; + DynamicFilter.Type localType = DynamicFilter.Type.LOCAL; + + List filterNodes = findFilterNodeInStage(planNode); + if (filterNodes.isEmpty()) { + buildNode = planNode.getRight(); + mapProbeSymbolsFromCriteria(planNode.getDynamicFilters(), localProbeSymbols, planNode.getCriteria()); + localType = DynamicFilter.Type.GLOBAL; + } + else { + buildNode = planNode.getRight(); + for (FilterNode filterNode : filterNodes) { + mapProbeSymbols(filterNode.getPredicate(), joinDynamicFilters, localProbeSymbols); + } + } + + final List buildSideSymbols = buildNode.getOutputSymbols(); + + Map localBuildChannels = planNode + .getDynamicFilters() + .entrySet() + .stream() + // Skip build channels that don't match local probe dynamic filters. + .filter(entry -> localProbeSymbols.containsKey(entry.getKey())) + .collect(toMap( + // Dynamic filter ID + entry -> entry.getKey(), + // Build-side channel index + entry -> { + Symbol buildSymbol = entry.getValue(); + int buildChannelIndex = buildSideSymbols.indexOf(buildSymbol); + verify(buildChannelIndex >= 0); + return buildChannelIndex; + })); + + if (localBuildChannels.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new LocalDynamicFilter(localProbeSymbols, localBuildChannels, partitionCount, localType, session, taskId, stateStoreProvider)); + } + public static Optional create(SemiJoinNode semiJoinNode, Session session, TaskId taskId, StateStoreProvider stateStoreProvider) { if (!semiJoinNode.getDynamicFilterId().isPresent()) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/io/prestosql/sql/planner/LocalExecutionPlanner.java index 598a1b9ab9402e0d4b61f7fcd19bf5ab8fabbc2c..56812c177828bf8b494b2dfc4aaf47aaeff21055 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/LocalExecutionPlanner.java @@ -64,7 +64,9 @@ import io.prestosql.operator.ExchangeOperator.ExchangeOperatorFactory; import io.prestosql.operator.ExplainAnalyzeOperator.ExplainAnalyzeOperatorFactory; import io.prestosql.operator.FilterAndProjectOperator; import io.prestosql.operator.GroupIdOperator; +import io.prestosql.operator.GroupJoinAggregator; import io.prestosql.operator.HashAggregationOperator.HashAggregationOperatorFactory; +import io.prestosql.operator.HashBuilderGroupJoinOperatorFactory; import io.prestosql.operator.HashBuilderOperator.HashBuilderOperatorFactory; import io.prestosql.operator.HashSemiJoinOperator.HashSemiJoinOperatorFactory; import io.prestosql.operator.JoinBridgeManager; @@ -124,6 +126,7 @@ import io.prestosql.operator.exchange.LocalExchangeSinkOperator.LocalExchangeSin import io.prestosql.operator.exchange.LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory; import io.prestosql.operator.exchange.LocalMergeSourceOperator.LocalMergeSourceOperatorFactory; import io.prestosql.operator.exchange.PageChannelSelector; +import io.prestosql.operator.groupjoin.ExecutionHelperFactory; import io.prestosql.operator.index.DynamicTupleFilterFactory; import io.prestosql.operator.index.FieldSetFilteringRecordSet; import io.prestosql.operator.index.IndexBuildDriverFactoryProvider; @@ -160,6 +163,8 @@ import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; +import io.prestosql.spi.plan.JoinOnAggregationNode.JoinInternalAggregation; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.OrderingScheme; @@ -331,6 +336,7 @@ import static io.prestosql.spi.util.Reflection.constructorMethodHandle; import static io.prestosql.spiller.PartitioningSpillerFactory.unsupportedPartitioningSpillerFactory; import static io.prestosql.sql.DynamicFilters.extractStaticFilters; import static io.prestosql.sql.gen.LambdaBytecodeGenerator.compileLambdaProvider; +import static io.prestosql.sql.planner.LocalExecutionPlanner.PhysicalOperation.toTypes; import static io.prestosql.sql.planner.RowExpressionInterpreter.Level.OPTIMIZED; import static io.prestosql.sql.planner.SystemPartitioningHandle.COORDINATOR_DISTRIBUTION; import static io.prestosql.sql.planner.SystemPartitioningHandle.FIXED_ARBITRARY_DISTRIBUTION; @@ -382,6 +388,7 @@ public class LocalExecutionPlanner protected final SpillerFactory spillerFactory; protected final SingleStreamSpillerFactory singleStreamSpillerFactory; protected final PartitioningSpillerFactory partitioningSpillerFactory; + protected final ExecutionHelperFactory executionHelperFactory; protected final PagesIndex.Factory pagesIndexFactory; protected final JoinCompiler joinCompiler; protected final LookupJoinOperators lookupJoinOperators; @@ -582,7 +589,8 @@ public class LocalExecutionPlanner ExchangeManagerRegistry exchangeManagerRegistry, TableExecuteContextManager tableExecuteContextManager, CachedDataManager cachedDataManager, - HetuConfig hetuConfig) + HetuConfig hetuConfig, + ExecutionHelperFactory executionHelperFactory) { this.explainAnalyzeContext = requireNonNull(explainAnalyzeContext, "explainAnalyzeContext is null"); this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); @@ -620,6 +628,7 @@ public class LocalExecutionPlanner this.tableExecuteContextManager = requireNonNull(tableExecuteContextManager, "tableExecuteContextManager is null"); this.cachedDataManager = requireNonNull(cachedDataManager, "cachedDataManager is null"); this.userName = requireNonNull(hetuConfig, "hetuConfig is null").getCachingUserName(); + this.executionHelperFactory = requireNonNull(executionHelperFactory, "executionHelperFactory is null"); } public LocalExecutionPlan plan( @@ -2301,6 +2310,25 @@ public class LocalExecutionPlanner } } + @Override + public PhysicalOperation visitJoinOnAggregation(JoinOnAggregationNode node, LocalExecutionPlanContext context) + { + List clauses = node.getCriteria(); + if (!node.getDynamicFilters().isEmpty()) { + log.debug("[GroupJoin] Dynamic filters: %s", node.getDynamicFilters()); + } + + List leftSymbols = Lists.transform(clauses, JoinNode.EquiJoinClause::getLeft); + List rightSymbols = Lists.transform(clauses, JoinNode.EquiJoinClause::getRight); + + switch (node.getType()) { + case INNER: + return createLookupGroupJoin(node, node.getLeft(), leftSymbols, node.getLeftHashSymbol(), node.getRight(), rightSymbols, node.getRightHashSymbol(), context); + default: + throw new UnsupportedOperationException("Unsupported join type: " + node.getType()); + } + } + @Override public PhysicalOperation visitSpatialJoin(SpatialJoinNode node, LocalExecutionPlanContext context) { @@ -2661,6 +2689,36 @@ public class LocalExecutionPlanner return new PhysicalOperation(operator, outputMappings.build(), context, probeSource); } + private PhysicalOperation createLookupGroupJoin(JoinOnAggregationNode node, + PlanNode probeNode, + List probeSymbols, + Optional probeHashSymbol, + PlanNode buildNode, + List buildSymbols, + Optional buildHashSymbol, + LocalExecutionPlanContext context) + { + // Plan probe + PhysicalOperation probeSource = probeNode.accept(this, context); + + // Plan build + ImmutableList.Builder finalOutputBuildTypes = ImmutableList.builder(); + ImmutableList.Builder buildFinalOutputChannels = ImmutableList.builder(); + JoinBridgeManager lookupSourceFactory = + createGroupLookupSourceFactory(node, buildNode, buildSymbols, buildHashSymbol, probeSource, context, finalOutputBuildTypes, buildFinalOutputChannels); + + OperatorFactory operator = createGroupLookupJoin(node, probeSource, probeSymbols, probeHashSymbol, lookupSourceFactory, context, finalOutputBuildTypes.build(), buildFinalOutputChannels.build()); + + ImmutableMap.Builder outputMappings = ImmutableMap.builder(); + List outputSymbols = node.getOutputSymbols(); + for (int i = 0; i < outputSymbols.size(); i++) { + Symbol symbol = outputSymbols.get(i); + outputMappings.put(symbol, i); + } + + return new PhysicalOperation(operator, outputMappings.build(), context, probeSource); + } + protected Optional createDynamicFilter(JoinNode node, LocalExecutionPlanContext context, int partitionCount) { if (!isEnableDynamicFiltering(context.getSession())) { @@ -2680,6 +2738,25 @@ public class LocalExecutionPlanner }); } + protected Optional createDynamicFilter(JoinOnAggregationNode node, LocalExecutionPlanContext context, int partitionCount) + { + if (!isEnableDynamicFiltering(context.getSession())) { + return Optional.empty(); + } + if (node.getDynamicFilters().isEmpty()) { + return Optional.empty(); + } + LocalDynamicFiltersCollector collector = context.getDynamicFiltersCollector(); + return LocalDynamicFilter + .create(node, partitionCount, context.getSession(), context.taskContext.getTaskId(), stateStoreProvider) + .map(filter -> { + // Intersect dynamic filters' predicates when they become ready, + // in order to support multiple join nodes in the same plan fragment. + addSuccessCallback(filter.getDynamicFilterResultFuture(), collector::intersectDynamicFilter); + return filter; + }); + } + private Optional createDynamicFilter(SemiJoinNode node, LocalExecutionPlanContext context) { if (!node.getDynamicFilterId().isPresent()) { @@ -2829,6 +2906,171 @@ public class LocalExecutionPlanner return lookupSourceFactoryManager; } + private JoinBridgeManager createGroupLookupSourceFactory( + JoinOnAggregationNode node, + PlanNode buildNode, + List buildSymbols, + Optional buildHashSymbol, + PhysicalOperation probeSource, + LocalExecutionPlanContext context, + ImmutableList.Builder finalOutputBuildTypes, + ImmutableList.Builder buildFinalOutputChannelsBuilder) + { + JoinInternalAggregation aggregationBuild = node.getRightAggr(); + LocalExecutionPlanContext buildContext = context.createSubContext(); + PhysicalOperation buildSource = buildNode.accept(this, buildContext); + if (buildSource.getPipelineExecutionStrategy() == GROUPED_EXECUTION) { + checkState( + probeSource.getPipelineExecutionStrategy() == GROUPED_EXECUTION, + "Build execution is GROUPED_EXECUTION. Probe execution is expected be GROUPED_EXECUTION, but is UNGROUPED_EXECUTION."); + } + + ImmutableMap.Builder buildAggrOutputMappings = ImmutableMap.builder(); + List aggrNodeOutputSymbols = aggregationBuild.getOutputSymbols(); + for (int i = 0; i < aggrNodeOutputSymbols.size(); i++) { + buildAggrOutputMappings.put(aggrNodeOutputSymbols.get(i), i); + } + ImmutableMap buildAggrLayout = buildAggrOutputMappings.build(); + List buildAggrTypes = toTypes(buildAggrLayout, buildContext); + + List buildTypes = toTypes(buildSource.getLayout(), buildContext); + + JoinInternalAggregation aggregationProbe = node.getLeftAggr(); + ImmutableMap.Builder probeAggrOutputMappings = ImmutableMap.builder(); + List probeAggrNodeOutputSymbols = aggregationProbe.getOutputSymbols(); + for (int i = 0; i < probeAggrNodeOutputSymbols.size(); i++) { + probeAggrOutputMappings.put(probeAggrNodeOutputSymbols.get(i), i); + } + ImmutableMap probeAggrLayout = probeAggrOutputMappings.build(); + + List buildAggrOutputSymbols = ImmutableList.copyOf(aggregationBuild.getOutputSymbols()); + List buildAggrOutputChannels = ImmutableList.copyOf(getChannelsForSymbols(buildAggrOutputSymbols, buildAggrLayout)); + List buildJoinChannels = ImmutableList.copyOf(getChannelsForSymbols(buildSymbols, buildAggrLayout)); + OptionalInt buildJoinHashChannel = buildHashSymbol.map(channelGetter(buildAggrLayout)) + .map(OptionalInt::of).orElse(OptionalInt.empty()); + int taskCount = buildContext.getDriverInstanceCount().orElse(1); + + Optional filterFunctionFactory = node.getFilter() + .map(filterExpression -> compileJoinFilterFunction( + filterExpression, + probeAggrLayout, + buildAggrLayout, + context.getTypes(), + context.getSession())); + + Optional sortExpressionContext = node.getFilter().flatMap(filter -> SortExpressionExtractor.extractSortExpression(metadata, node.getRightOutputSymbols(), filter)); + + Optional sortChannel = sortExpressionContext + .map(SortExpressionContext::getSortExpression) + .map(sortExpression -> sortExpressionAsSortChannel(sortExpression, probeAggrLayout, buildAggrLayout, context)); + + List searchFunctionFactories = sortExpressionContext + .map(SortExpressionContext::getSearchExpressions) + .map(searchExpressions -> searchExpressions.stream() + .map(searchExpression -> compileJoinFilterFunction( + searchExpression, + probeAggrLayout, + buildAggrLayout, + context.getTypes(), + context.getSession())) + .collect(toImmutableList())) + .orElse(ImmutableList.of()); + + ImmutableList buildAggrOutputTypes = buildAggrOutputChannels.stream() + .map(buildAggrTypes::get) + .collect(toImmutableList()); + JoinBridgeManager lookupSourceFactoryManager = new JoinBridgeManager<>( + false, + probeSource.getPipelineExecutionStrategy(), + buildSource.getPipelineExecutionStrategy(), + lifespan -> new PartitionedLookupSourceFactory( + buildAggrTypes, + buildAggrOutputTypes, + buildJoinChannels.stream() + .map(buildAggrTypes::get) + .collect(toImmutableList()), + taskCount, + buildAggrLayout, + false, + false), + buildAggrOutputTypes); + + ImmutableList.Builder factoriesBuilder = new ImmutableList.Builder(); + factoriesBuilder.addAll(buildSource.getOperatorFactories()); + + createDynamicFilter(node, context, taskCount).ifPresent( + filter -> { + List filterBuildChannels = filter + .getBuildChannels() + .entrySet() + .stream() + .map(entry -> { + String filterId = entry.getKey(); + int index = entry.getValue(); + Type type = buildAggrTypes.get(index); + return new DynamicFilterSourceOperator.Channel(filterId, type, index, context.getSession().getQueryId().toString()); + }) + .collect(Collectors.toList()); + factoriesBuilder.add( + new DynamicFilterSourceOperator.DynamicFilterSourceOperatorFactory( + buildContext.getNextOperatorId(), + node.getId(), + filter.getValueConsumer(), /** the consumer to process all values collected to build the dynamic filter */ + filterBuildChannels, + getDynamicFilteringMaxPerDriverValueCount(buildContext.getSession()), + getDynamicFilteringMaxPerDriverSize(buildContext.getSession()))); + }); + + int startOutputChannel = 0; + ImmutableMap.Builder outputMappings = ImmutableMap.builder(); + ImmutableMap.Builder finalOutputMappings = ImmutableMap.builder(); + GroupJoinAggregator aggrfactory = prepareGroupJoinAggregator(aggregationBuild, + buildSource.getLayout(), + buildSource.getTypes(), + startOutputChannel, + outputMappings); + GroupJoinAggregator aggrOnAggrfactory = prepareGroupJoinAggregator(node.getAggrOnRight(), + buildAggrLayout, + buildAggrTypes, + startOutputChannel, + finalOutputMappings); + List buildFinalOutputSymbols = finalOutputMappings.build().keySet().stream() + .filter(symbol -> node.getOutputSymbols().contains(symbol)) + .collect(toImmutableList()); + List buildFinalOutputChannels = ImmutableList.copyOf(getChannelsForSymbols(buildFinalOutputSymbols, finalOutputMappings.build())); + buildFinalOutputChannelsBuilder.addAll(buildFinalOutputChannels); + finalOutputBuildTypes.addAll(toTypes(buildFinalOutputSymbols, buildContext)); + HashBuilderGroupJoinOperatorFactory hashBuilderOperatorFactory = new HashBuilderGroupJoinOperatorFactory( + buildContext.getNextOperatorId(), + node.getId(), + lookupSourceFactoryManager, + buildAggrOutputChannels, + buildJoinChannels, + buildJoinHashChannel, + filterFunctionFactory, + sortChannel, + Optional.of(buildAggrLayout.size() - 1), + searchFunctionFactories, + 10_000, + pagesIndexFactory, + taskCount > 1, + isSpillToHdfsEnabled(context.getSession()), + aggrfactory, + aggrOnAggrfactory, + executionHelperFactory); + + factoriesBuilder.add(hashBuilderOperatorFactory); + + context.addDriverFactory( + buildContext.isInputDriver(), + false, + factoriesBuilder.build(), + buildContext.getDriverInstanceCount(), + buildSource.getPipelineExecutionStrategy()); + + return lookupSourceFactoryManager; + } + protected JoinFilterFunctionFactory compileJoinFilterFunction( RowExpression filterExpression, Map probeLayout, @@ -2852,6 +3094,122 @@ public class LocalExecutionPlanner return ((InputReferenceExpression) rewrittenSortExpression).getField(); } + private OperatorFactory createGroupLookupJoin( + JoinOnAggregationNode node, + PhysicalOperation probeSource, + List probeSymbols, + Optional probeHashSymbol, + JoinBridgeManager lookupSourceFactoryManager, + LocalExecutionPlanContext context, + List finalOutputBuildTypes, + List buildFinalOutputChannels) + { + // Build output depends on Left Aggregation + JoinInternalAggregation aggregationProbe = node.getLeftAggr(); + ImmutableMap.Builder probeAggrOutputMappings = ImmutableMap.builder(); + List aggrnodeOutputSymbols = aggregationProbe.getOutputSymbols(); + for (int i = 0; i < aggrnodeOutputSymbols.size(); i++) { + probeAggrOutputMappings.put(aggrnodeOutputSymbols.get(i), i); + } + ImmutableMap probeAggrLayout = probeAggrOutputMappings.build(); + LocalExecutionPlanContext probeContext = context.createSubContext(); + + List probeAggrTypes = toTypes(probeAggrLayout, probeContext); + List probeAggrOutputSymbols = ImmutableList.copyOf(aggregationProbe.getOutputSymbols()); + List probeAggrOutputChannels = ImmutableList.copyOf(getChannelsForSymbols(probeAggrOutputSymbols, probeAggrLayout)); + List probeJoinChannels = ImmutableList.copyOf(getChannelsForSymbols(probeSymbols, probeAggrLayout)); + OptionalInt probeJoinHashChannel = probeHashSymbol.map(channelGetter(probeAggrLayout)) + .map(OptionalInt::of).orElse(OptionalInt.empty()); + OptionalInt totalOperatorsCount = context.getDriverInstanceCount(); + OptionalInt probeAggrCountChannel = OptionalInt.of(probeAggrOutputChannels.size() - 1); + + int startOutputChannel = 0; + ImmutableMap.Builder outputMappings = ImmutableMap.builder(); + ImmutableMap.Builder finalOutputMappings = ImmutableMap.builder(); + + GroupJoinAggregator aggrfactory = prepareGroupJoinAggregator(aggregationProbe, + probeSource.getLayout(), + probeSource.getTypes(), + startOutputChannel, + outputMappings); + GroupJoinAggregator aggrOnAggrfactory = prepareGroupJoinAggregator(node.getAggrOnLeft(), + probeAggrLayout, + probeAggrTypes, + startOutputChannel, + finalOutputMappings); + + List probeFinalOutputSymbols = finalOutputMappings.build().keySet().stream() + .filter(symbol -> node.getOutputSymbols().contains(symbol)) + .collect(toImmutableList()); + List probeFinalOutputChannels = ImmutableList.copyOf(getChannelsForSymbols(probeFinalOutputSymbols, finalOutputMappings.build())); + + ImmutableList.Builder outputTypes = ImmutableList.builder(); + outputTypes.addAll(toTypes(probeFinalOutputSymbols, probeContext)).addAll(finalOutputBuildTypes); + + switch (node.getType()) { + case INNER: + return lookupJoinOperators.groupInnerJoin(context.getNextOperatorId(), + node.getId(), + lookupSourceFactoryManager, + probeAggrTypes, + probeJoinChannels, + probeJoinHashChannel, + probeAggrCountChannel, + probeAggrOutputChannels, + totalOperatorsCount, + partitioningSpillerFactory, + false, + aggrfactory, + aggrOnAggrfactory, + probeFinalOutputChannels, + buildFinalOutputChannels, + outputTypes.build(), + executionHelperFactory, + 10_000, + pagesIndexFactory); + default: + throw new UnsupportedOperationException("Unsupported join type: " + node.getType()); + } + } + + private GroupJoinAggregator prepareGroupJoinAggregator(JoinInternalAggregation aggregation, + Map layout, + List types, + int startOutputChannel, + ImmutableMap.Builder outputMappings) + { + List accumulatorFactories = new ArrayList<>(); + List groupByChannels = getChannelsForSymbols(aggregation.getGroupingKeys(), layout); + List groupByTypes = groupByChannels.stream() + .map(types::get) + .collect(toImmutableList()); + Optional hashChannel = aggregation.getHashSymbol().map(channelGetter(layout)); + Optional groupIdChannel = getOutputMappingAndGroupIdChannel(aggregation.getAggregations(), + aggregation.getGroupingKeys(), + aggregation.getHashSymbol(), + aggregation.getGroupIdSymbol(), + layout, + types, + startOutputChannel, + outputMappings, + accumulatorFactories, + Optional.empty(), + Optional.empty()); + return GroupJoinAggregator.buildGroupJoinAggregator(groupByTypes, + groupByChannels, + ImmutableList.copyOf(aggregation.getGlobalGroupingSets()), + aggregation.getStep(), + accumulatorFactories, + hashChannel, + groupIdChannel, + 10_000, + Optional.ofNullable(maxPartialAggregationMemorySize), + joinCompiler, + aggregation.getStep().isOutputPartial(), + createPartialAggregationController(aggregation.getStep(), session), + aggregation.hasDefaultOutput()); + } + private OperatorFactory createLookupJoin( JoinNode node, PhysicalOperation probeSource, @@ -3545,6 +3903,14 @@ public class LocalExecutionPlanner protected AccumulatorFactory buildAccumulatorFactory( PhysicalOperation source, Aggregation aggregation) + { + return buildAccumulatorFactory(source.getLayout(), source.getTypes(), aggregation); + } + + protected AccumulatorFactory buildAccumulatorFactory( + Map layout, + List types, + Aggregation aggregation) { InternalAggregationFunction internalAggregationFunction = metadata.getFunctionAndTypeManager().getAggregateFunctionImplementation(aggregation.getFunctionHandle()); @@ -3552,7 +3918,7 @@ public class LocalExecutionPlanner for (RowExpression argument : aggregation.getArguments()) { if (!(argument instanceof LambdaDefinitionExpression)) { checkArgument(argument instanceof VariableReferenceExpression, "argument must be variable reference"); - valueChannels.add(source.getLayout().get(new Symbol(((VariableReferenceExpression) argument).getName()))); + valueChannels.add(layout.get(new Symbol(((VariableReferenceExpression) argument).getName()))); } } @@ -3572,7 +3938,7 @@ public class LocalExecutionPlanner } } - Optional maskChannel = aggregation.getMask().map(value -> source.getLayout().get(value)); + Optional maskChannel = aggregation.getMask().map(value -> layout.get(value)); List sortOrders = ImmutableList.of(); List sortKeys = ImmutableList.of(); if (aggregation.getOrderingScheme().isPresent()) { @@ -3586,8 +3952,8 @@ public class LocalExecutionPlanner return internalAggregationFunction.bind( valueChannels, maskChannel, - source.getTypes(), - getChannelsForSymbols(sortKeys, source.getLayout()), + types, + getChannelsForSymbols(sortKeys, layout), sortOrders, pagesIndexFactory, aggregation.isDistinct(), @@ -3804,11 +4170,36 @@ public class LocalExecutionPlanner createPartialAggregationController(step, session)); } + private Optional getOutputMappingAndGroupIdChannel(Map aggregations, + List groupBySymbols, + Optional hashSymbol, + Optional groupIdSymbol, + PhysicalOperation source, + int startOutputChannel, + ImmutableMap.Builder outputMappings, + List accumulatorFactories, + Optional step, + Optional finalizeSymbol) + { + return getOutputMappingAndGroupIdChannel(aggregations, + groupBySymbols, + hashSymbol, + groupIdSymbol, + source.getLayout(), + source.getTypes(), + startOutputChannel, + outputMappings, + accumulatorFactories, + step, + finalizeSymbol); + } + private Optional getOutputMappingAndGroupIdChannel(Map aggregations, List groupBySymbols, Optional hashSymbol, Optional groupIdSymbol, - PhysicalOperation source, + Map layout, + List types, int startOutputChannel, ImmutableMap.Builder outputMappings, List accumulatorFactories, @@ -3819,7 +4210,7 @@ public class LocalExecutionPlanner for (Map.Entry entry : aggregations.entrySet()) { Symbol symbol = entry.getKey(); Aggregation aggregation = entry.getValue(); - accumulatorFactories.add(buildAccumulatorFactory(source, aggregation)); + accumulatorFactories.add(buildAccumulatorFactory(layout, types, aggregation)); aggregationOutputSymbols.add(symbol); } @@ -3937,10 +4328,15 @@ public class LocalExecutionPlanner } protected static Function channelGetter(PhysicalOperation source) + { + return channelGetter(source.getLayout()); + } + + protected static Function channelGetter(Map layout) { return input -> { - checkArgument(source.getLayout().containsKey(input)); - return source.getLayout().get(input); + checkArgument(layout.containsKey(input)); + return layout.get(input); }; } @@ -3987,7 +4383,18 @@ public class LocalExecutionPlanner this.pipelineExecutionStrategy = pipelineExecutionStrategy; } - private static List toTypes(Map layout, LocalExecutionPlanContext context) + public static List toTypes(List symbols, LocalExecutionPlanContext context) + { + ImmutableList.Builder builder = ImmutableList.builder(); + symbols.forEach(symbol -> { + Type type = context.getTypes().get(symbol); + checkArgument(type != null, "Layout does not have a symbol for every output channel: %s", symbol); + builder.add(type); + }); + return builder.build(); + } + + public static List toTypes(Map layout, LocalExecutionPlanContext context) { // verify layout covers all values int channelCount = layout.values().stream().mapToInt(Integer::intValue).max().orElse(-1) + 1; diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/PlanFragmenter.java b/presto-main/src/main/java/io/prestosql/sql/planner/PlanFragmenter.java index 437c989fda0eb9257858a7515d8b330f1c6ed6d4..259663c9e7a2b317843d4c46c19c671938d73f93 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/PlanFragmenter.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/PlanFragmenter.java @@ -33,6 +33,7 @@ import io.prestosql.spi.metadata.TableHandle; import io.prestosql.spi.plan.AggregationNode; import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeId; import io.prestosql.spi.plan.ProjectNode; @@ -761,6 +762,71 @@ public class PlanFragmenter } } + @Override + public GroupedExecutionProperties visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + GroupedExecutionProperties left = node.getLeft().accept(this, null); + GroupedExecutionProperties right = node.getRight().accept(this, null); + + if (!node.getDistributionType().isPresent()) { + // This is possible when the optimizers is invoked with `forceSingleNode` set to true. + return GroupedExecutionProperties.notCapable(); + } + + if ((node.getType() == JoinNode.Type.RIGHT || node.getType() == JoinNode.Type.FULL) && !right.currentNodeCapable) { + // For a plan like this, if the fragment participates in grouped execution, + // the LookupOuterOperator corresponding to the RJoin will not work execute properly. + // + // * The operator has to execute as not-grouped because it can only look at the "used" flags in + // join build after all probe has finished. + // * The operator has to execute as grouped the subsequent LJoin expects that incoming + // operators are grouped. Otherwise, the LJoin won't be able to throw out the build side + // for each group as soon as the group completes. + // + // LJoin + // / \ + // RJoin Scan + // / \ + // Scan Remote + // + // TODO: + // The RJoin can still execute as grouped if there is no subsequent operator that depends + // on the RJoin being executed in a grouped manner. However, this is not currently implemented. + // Support for this scenario is already implemented in the execution side. + return GroupedExecutionProperties.notCapable(); + } + + switch (node.getDistributionType().get()) { + case REPLICATED: + // Broadcast join maintains partitioning for the left side. + // Right side of a broadcast is not capable of grouped execution because it always comes from a remote exchange. + checkState(!right.currentNodeCapable); + return left; + case PARTITIONED: + if (left.currentNodeCapable && right.currentNodeCapable) { + return new GroupedExecutionProperties( + true, + true, + ImmutableList.builder() + .addAll(left.capableTableScanNodes) + .addAll(right.capableTableScanNodes) + .build()); + } + // right.subTreeUseful && !left.currentNodeCapable: + // It's not particularly helpful to do grouped execution on the right side + // because the benefit is likely cancelled out due to required buffering for hash build. + // In theory, it could still be helpful (e.g. when the underlying aggregation's intermediate group state maybe larger than aggregation output). + // However, this is not currently implemented. JoinBridgeManager need to support such a lifecycle. + // !right.currentNodeCapable: + // The build/right side needs to buffer fully for this JOIN, but the probe/left side will still stream through. + // As a result, there is no reason to change currentNodeCapable or subTreeUseful to false. + // + return left; + default: + throw new UnsupportedOperationException("Unknown distribution type: " + node.getDistributionType()); + } + } + @Override public GroupedExecutionProperties visitAggregation(AggregationNode node, Void context) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/io/prestosql/sql/planner/PlanOptimizers.java index 9df53f30c0b5d13ced1b2a7a56334bcb84df57cc..cc206a925f243df6746929c95e8c58b6f1045669 100755 --- a/presto-main/src/main/java/io/prestosql/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/PlanOptimizers.java @@ -63,6 +63,8 @@ import io.prestosql.sql.planner.iterative.rule.MergeLimitWithDistinct; import io.prestosql.sql.planner.iterative.rule.MergeLimitWithSort; import io.prestosql.sql.planner.iterative.rule.MergeLimitWithTopN; import io.prestosql.sql.planner.iterative.rule.MergeLimits; +import io.prestosql.sql.planner.iterative.rule.MergePartialAggregationWithJoin; +import io.prestosql.sql.planner.iterative.rule.MergePartialAggregationWithJoinPushProject; import io.prestosql.sql.planner.iterative.rule.MultipleDistinctAggregationToMarkDistinct; import io.prestosql.sql.planner.iterative.rule.PruneAggregationColumns; import io.prestosql.sql.planner.iterative.rule.PruneAggregationSourceColumns; @@ -762,7 +764,9 @@ public class PlanOptimizers ImmutableSet.of( new PushPartialAggregationThroughJoin(), new PushPartialAggregationThroughExchange(metadata), - new PruneJoinColumns()))); + new PruneJoinColumns(), + new MergePartialAggregationWithJoin(metadata), + new MergePartialAggregationWithJoinPushProject(metadata)))); builder.add(new IterativeOptimizer( ruleStats, diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/PlanSymbolAllocator.java b/presto-main/src/main/java/io/prestosql/sql/planner/PlanSymbolAllocator.java index af0346601980498780a8db969a86310b25c540e8..426e8a45d791957e73191a9e1035fd9bec1b38fa 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/PlanSymbolAllocator.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/PlanSymbolAllocator.java @@ -166,4 +166,9 @@ public class PlanSymbolAllocator { return nextId++; } + + public void updateSymbolType(Symbol symbol, Type newReturnType) + { + symbols.put(symbol, newReturnType); + } } diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/SchedulingOrderVisitor.java b/presto-main/src/main/java/io/prestosql/sql/planner/SchedulingOrderVisitor.java index 722b3302c573633da96d8c8843043bcb1be61775..f8a055ad58a603cae6869a707f7e9b1604ad161b 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/SchedulingOrderVisitor.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/SchedulingOrderVisitor.java @@ -16,6 +16,7 @@ package io.prestosql.sql.planner; import com.google.common.collect.ImmutableList; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeId; import io.prestosql.spi.plan.TableScanNode; @@ -59,6 +60,14 @@ public class SchedulingOrderVisitor return null; } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Consumer schedulingOrder) + { + node.getRight().accept(this, schedulingOrder); + node.getLeft().accept(this, schedulingOrder); + return null; + } + @Override public Void visitSemiJoin(SemiJoinNode node, Consumer schedulingOrder) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/SplitSourceFactory.java b/presto-main/src/main/java/io/prestosql/sql/planner/SplitSourceFactory.java index c12a17892942560a731a2eca42115d381d7b8038..fb32fe9af6b0f3c08d1c09a4d29bf23bcba6e8fa 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/SplitSourceFactory.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/SplitSourceFactory.java @@ -23,6 +23,7 @@ import io.prestosql.spi.plan.AggregationNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; @@ -177,6 +178,17 @@ public class SplitSourceFactory .build(); } + @Override + public Map visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + Map leftSplits = node.getLeft().accept(this, context); + Map rightSplits = node.getRight().accept(this, context); + return ImmutableMap.builder() + .putAll(leftSplits) + .putAll(rightSplits) + .build(); + } + @Override public Map visitSemiJoin(SemiJoinNode node, Void context) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/HintedReorderJoins.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/HintedReorderJoins.java index 065b15316f4ebcc652f330099f8e984db83f7b09..668f25627161ceff3ce336f7a19c06d307b0258c 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/HintedReorderJoins.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/HintedReorderJoins.java @@ -40,6 +40,7 @@ import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; import io.prestosql.spi.plan.JoinNode.DistributionType; import io.prestosql.spi.plan.JoinNode.EquiJoinClause; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeIdAllocator; import io.prestosql.spi.plan.Symbol; @@ -217,25 +218,43 @@ public class HintedReorderJoins private void flattenNode(PlanNode node, int limit) { // (limit - 2) because you need to account for adding left and right side - if (!(node instanceof JoinNode) || (sources.size() > (limit - 2))) { + if (!(node instanceof JoinNode || node instanceof JoinOnAggregationNode) || (sources.size() > (limit - 2))) { sources.add(node); return; } - JoinNode joinNode = (JoinNode) node; - if (joinNode.getType() != INNER - || !logicalRowExpressions.isDeterministic(joinNode.getFilter().orElse(TRUE_CONSTANT)) - || joinNode.getDistributionType().isPresent()) { - sources.add(node); - return; - } + if (node instanceof JoinNode) { + JoinNode joinNode = (JoinNode) node; + if (joinNode.getType() != INNER + || !logicalRowExpressions.isDeterministic(joinNode.getFilter().orElse(TRUE_CONSTANT)) + || joinNode.getDistributionType().isPresent()) { + sources.add(node); + return; + } + + // we set the left limit to limit - 1 to account for the node on the right + flattenNode(joinNode.getLeft(), limit - 1); + flattenNode(joinNode.getRight(), limit); + joinNode.getCriteria().stream() + .map(criteria -> toRowExpression(criteria, types)) + .forEach(filters::add); + } + else if (node instanceof JoinOnAggregationNode) { + JoinOnAggregationNode joinNode = (JoinOnAggregationNode) node; + if (joinNode.getType() != INNER + || !logicalRowExpressions.isDeterministic(joinNode.getFilter().orElse(TRUE_CONSTANT)) + || joinNode.getDistributionType().isPresent()) { + sources.add(node); + return; + } - // we set the left limit to limit - 1 to account for the node on the right - flattenNode(joinNode.getLeft(), limit - 1); - flattenNode(joinNode.getRight(), limit); - joinNode.getCriteria().stream() - .map(criteria -> toRowExpression(criteria, types)) - .forEach(filters::add); + // we set the left limit to limit - 1 to account for the node on the right + flattenNode(joinNode.getLeft(), limit - 1); + flattenNode(joinNode.getRight(), limit); + joinNode.getCriteria().stream() + .map(criteria -> toRowExpression(criteria, types)) + .forEach(filters::add); + } } ReorderJoins.MultiJoinNode toMultiJoinNode() @@ -769,26 +788,44 @@ public class HintedReorderJoins PlanNode resolved = lookup.resolve(node); // (limit - 2) because you need to account for adding left and right side - if (!(resolved instanceof JoinNode) || (sources.size() > (limit - 2))) { + if (!(resolved instanceof JoinNode || resolved instanceof JoinOnAggregationNode) || (sources.size() > (limit - 2))) { sources.add(node); return; } + if (resolved instanceof JoinNode) { + JoinNode joinNode = (JoinNode) resolved; + if (joinNode.getType() != INNER + || !determinismEvaluator.isDeterministic(joinNode.getFilter().orElse(TRUE_CONSTANT)) + || joinNode.getDistributionType().isPresent()) { + sources.add(node); + return; + } - JoinNode joinNode = (JoinNode) resolved; - if (joinNode.getType() != INNER - || !determinismEvaluator.isDeterministic(joinNode.getFilter().orElse(TRUE_CONSTANT)) - || joinNode.getDistributionType().isPresent()) { - sources.add(node); - return; + // we set the left limit to limit - 1 to account for the node on the right + flattenNode(joinNode.getLeft(), limit - 1); + flattenNode(joinNode.getRight(), limit); + joinNode.getCriteria().stream() + .map(criteria -> toRowExpression(criteria, types)) + .forEach(filters::add); + joinNode.getFilter().ifPresent(filters::add); } + else if (resolved instanceof JoinOnAggregationNode) { + JoinOnAggregationNode joinNode = (JoinOnAggregationNode) resolved; + if (joinNode.getType() != INNER + || !determinismEvaluator.isDeterministic(joinNode.getFilter().orElse(TRUE_CONSTANT)) + || joinNode.getDistributionType().isPresent()) { + sources.add(node); + return; + } - // we set the left limit to limit - 1 to account for the node on the right - flattenNode(joinNode.getLeft(), limit - 1); - flattenNode(joinNode.getRight(), limit); - joinNode.getCriteria().stream() - .map(criteria -> toRowExpression(criteria, types)) - .forEach(filters::add); - joinNode.getFilter().ifPresent(filters::add); + // we set the left limit to limit - 1 to account for the node on the right + flattenNode(joinNode.getLeft(), limit - 1); + flattenNode(joinNode.getRight(), limit); + joinNode.getCriteria().stream() + .map(criteria -> toRowExpression(criteria, types)) + .forEach(filters::add); + joinNode.getFilter().ifPresent(filters::add); + } } MultiJoinNode toMultiJoinNode() @@ -916,6 +953,19 @@ public class HintedReorderJoins return null; } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, StringBuilder context) + { + PlanNode left = lookup.resolve(node.getLeft()); + PlanNode right = lookup.resolve(node.getRight()); + context.append('('); + left.accept(this, context); + context.append(','); + right.accept(this, context); + context.append(')'); + return null; + } + public static boolean startsWith(String actualPattern, String expectedPattern) { return expectedPattern.contains(actualPattern); diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/MergePartialAggregationWithJoin.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/MergePartialAggregationWithJoin.java new file mode 100644 index 0000000000000000000000000000000000000000..76b121f0b790af285d98cb6ec2559e558aee2b83 --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/MergePartialAggregationWithJoin.java @@ -0,0 +1,470 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.sql.planner.iterative.rule; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Streams; +import io.prestosql.Session; +import io.prestosql.matching.Capture; +import io.prestosql.matching.Captures; +import io.prestosql.matching.Pattern; +import io.prestosql.metadata.Metadata; +import io.prestosql.spi.function.BuiltInFunctionHandle; +import io.prestosql.spi.function.FunctionHandle; +import io.prestosql.spi.plan.AggregationNode; +import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinNode.EquiJoinClause; +import io.prestosql.spi.plan.JoinOnAggregationNode; +import io.prestosql.spi.plan.JoinOnAggregationNode.JoinInternalAggregation; +import io.prestosql.spi.plan.PlanNode; +import io.prestosql.spi.plan.PlanNodeId; +import io.prestosql.spi.plan.Symbol; +import io.prestosql.spi.relation.CallExpression; +import io.prestosql.spi.relation.VariableReferenceExpression; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeSignature; +import io.prestosql.sql.analyzer.TypeSignatureProvider; +import io.prestosql.sql.planner.SymbolsExtractor; +import io.prestosql.sql.planner.iterative.Rule; + +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.Sets.intersection; +import static io.prestosql.SystemSessionProperties.isMergePartialAggregationWithJoin; +import static io.prestosql.SystemSessionProperties.isSnapshotEnabled; +import static io.prestosql.SystemSessionProperties.isSpillEnabled; +import static io.prestosql.spi.plan.AggregationNode.Aggregation; +import static io.prestosql.spi.plan.AggregationNode.Step.PARTIAL; +import static io.prestosql.spi.plan.AggregationNode.singleGroupingSet; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.sql.planner.iterative.rule.MergePartialAggregationWithJoin.GroupJoinAggregationFunction.SUPPORTED_FUNCTIONS; +import static io.prestosql.sql.planner.iterative.rule.Util.restrictOutputs; +import static io.prestosql.sql.planner.plan.Patterns.aggregation; +import static io.prestosql.sql.planner.plan.Patterns.join; +import static io.prestosql.sql.planner.plan.Patterns.source; + +public class MergePartialAggregationWithJoin + implements Rule +{ + static final Capture JOIN_NODE = Capture.newCapture(); + + private static final Pattern PATTERN = aggregation() + .matching(MergePartialAggregationWithJoin::isSupportedAggregationNode) + .with(source().matching(join().capturedAs(JOIN_NODE))); + + private final Metadata metadata; + + public MergePartialAggregationWithJoin(Metadata metadata) + { + this.metadata = metadata; + } + + static boolean isSupportedAggregationNode(AggregationNode aggregationNode) + { + // Don't split streaming aggregations + if (aggregationNode.isStreamable()) { + return false; + } + + if (aggregationNode.getHashSymbol().isPresent()) { + // TODO: add support for hash symbol in aggregation node + return false; + } + return aggregationNode.getStep() == PARTIAL && aggregationNode.getGroupingSetCount() == 1; + } + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isMergePartialAggregationWithJoin(session) + && !isSpillEnabled(session) + && !isSnapshotEnabled(session); + } + + @Override + public Result apply(AggregationNode aggregationNode, Captures captures, Context context) + { + JoinNode joinNode = captures.get(JOIN_NODE); + + if (joinNode.getType() != JoinNode.Type.INNER) { + return Result.empty(); + } + + // Merge AggregationNode within JoinNode + return checkAndApplyRule(aggregationNode, context, joinNode); + } + + protected Result checkAndApplyRule(AggregationNode aggregationNode, Context context, JoinNode joinNode) + { + // Verify Aggregations against Whitelist functions + if (!checkAggregationsInWhitelist(aggregationNode.getAggregations())) { + return Result.empty(); + } + if (aggregationKeysSameAsJoinKeysOrSuperWithPresentInStart(aggregationNode.getGroupingKeys(), joinNode.getCriteria(), joinNode)) { + return Result.ofPlanNode(mergePartialAggregationWithJoin(aggregationNode, joinNode, context)); + } + + if (isJoinKeysSubsetOfAggrGroupKeys(aggregationNode.getGroupingKeys(), joinNode.getCriteria())) { + return Result.ofPlanNode(mergePartialAggregationWithJoin(aggregationNode, joinNode, context)); + } + + return Result.empty(); + } + + private boolean checkAggregationsInWhitelist(Map aggregations) + { + for (Map.Entry entry : aggregations.entrySet()) { + if (!SUPPORTED_FUNCTIONS.contains(entry.getValue().getFunctionCall().getDisplayName().toLowerCase(Locale.ROOT))) { + return false; + } + } + return true; + } + + private JoinInternalAggregation replaceAggregationSource( + PlanNodeId id, + AggregationNode aggregation, + Map aggregations, + PlanNode source, + List groupingKeys) + { + return new JoinInternalAggregation( + id, + source, + aggregations, + singleGroupingSet(groupingKeys), + ImmutableList.of(), + aggregation.getStep(), + aggregation.getHashSymbol(), + aggregation.getGroupIdSymbol(), + aggregation.getAggregationType(), + aggregation.getFinalizeSymbol()); + } + + private PlanNode mergePartialAggregationWithJoin(AggregationNode aggregationNode, JoinNode child, Context context) + { + // Divide AggregationNode into left and right AggregationNode + Set joinLeftChildSymbols = ImmutableSet.copyOf(child.getLeft().getOutputSymbols()); + List leftGroupingKeys = getPushedDownGroupingSet(aggregationNode, joinLeftChildSymbols, + intersection(getJoinRequiredSymbols(child), joinLeftChildSymbols)); + Map leftAggregations = getAggregationsMatchingSymbols( + aggregationNode.getAggregations(), joinLeftChildSymbols); + + Set joinRightChildSymbols = ImmutableSet.copyOf(child.getRight().getOutputSymbols()); + List rightGroupingKeys = getPushedDownGroupingSet(aggregationNode, joinRightChildSymbols, + intersection(getJoinRequiredSymbols(child), joinRightChildSymbols)); + Map rightAggregations = getAggregationsMatchingSymbols( + aggregationNode.getAggregations(), joinRightChildSymbols); + Map commonAggregations = getCommonAggregations(aggregationNode.getAggregations()); + if (leftAggregations.size() > 0) { + leftAggregations.putAll(commonAggregations); + } + else if (rightAggregations.size() > 0) { + rightAggregations.putAll(commonAggregations); + } + else { + leftAggregations.putAll(commonAggregations); + } + String aggFunction = "count"; + FunctionHandle functionHandle = metadata.getFunctionAndTypeManager().lookupFunction(aggFunction, + TypeSignatureProvider.fromTypeSignatures()); + CallExpression countExpr = new CallExpression( + aggFunction, + functionHandle, + BIGINT, + ImmutableList.of()); + Aggregation count = new Aggregation( + countExpr, + ImmutableList.of(), + false, + Optional.empty(), + Optional.empty(), + Optional.empty()); + Symbol countSymbolLeft = context.getSymbolAllocator().newSymbol(countExpr.getDisplayName(), BIGINT, null); + Symbol countSymbolRight = context.getSymbolAllocator().newSymbol(countExpr.getDisplayName(), BIGINT, null); + + ImmutableMap.Builder newLeftAggrs = ImmutableMap.builder(); + ImmutableMap.Builder newRightAggrs = ImmutableMap.builder(); + Map aggrOnLeftAggregations = getNewAggrOnAggr(leftAggregations, context, newLeftAggrs); + Map aggrOnRightAggregations = getNewAggrOnAggr(rightAggregations, context, newRightAggrs); + JoinInternalAggregation left = replaceAggregationSource(child.getId(), + aggregationNode, + newLeftAggrs.put(countSymbolLeft, count).build(), + child.getLeft(), + leftGroupingKeys); + JoinInternalAggregation right = replaceAggregationSource(child.getId(), + aggregationNode, + newRightAggrs.put(countSymbolRight, count).build(), + child.getRight(), + rightGroupingKeys); + + JoinInternalAggregation aggrOnLeft = replaceAggregationSource(child.getId(), + aggregationNode, + aggrOnLeftAggregations, + left, + leftGroupingKeys); + JoinInternalAggregation aggrOnRight = replaceAggregationSource(child.getId(), + aggregationNode, + aggrOnRightAggregations, + right, + rightGroupingKeys); + + ImmutableList.Builder outputSymbols = ImmutableList.builder(); + outputSymbols.addAll(aggrOnLeft.getOutputSymbols()).addAll(aggrOnRight.getOutputSymbols()); + JoinOnAggregationNode newJoinNode = new JoinOnAggregationNode(child.getId(), + child.getType(), + child.getCriteria(), + child.getFilter(), + child.getLeftHashSymbol(), + child.getRightHashSymbol(), + child.getDistributionType(), + child.isSpillable(), + child.getDynamicFilters(), + left, + right, + aggrOnLeft, + aggrOnRight, + outputSymbols.build()); + return restrictOutputs(context.getIdAllocator(), + newJoinNode, + ImmutableSet.copyOf(aggregationNode.getOutputSymbols()), + true, + context.getSymbolAllocator().getTypes()).orElse(newJoinNode); + } + + private Map getNewAggrOnAggr(Map aggregations, + Context context, + Builder origAggrsNewRef) + { + String aggFunctionSum = "sum"; + + ImmutableMap.Builder newAggregations = ImmutableMap.builder(); + aggregations.forEach((symbol, aggregation) -> { + CallExpression expression = aggregation.getFunctionCall(); + CallExpression newExpr = null; + + // Define a new symbol for original expression + TypeSignature origAggrReturnTypeSig = ((BuiltInFunctionHandle) expression.getFunctionHandle()).getSignature().getReturnType(); + Type origAggrReturnType = metadata.getFunctionAndTypeManager().getType(origAggrReturnTypeSig); + Symbol newSymbol = null; + + if (expression.getDisplayName().equals("count")) { + FunctionHandle newFunctionHandle = metadata.getFunctionAndTypeManager().lookupFunction(aggFunctionSum, + TypeSignatureProvider.fromTypeSignatures(origAggrReturnTypeSig)); + + // newSymbol is going to point to origExpression, so using its returnType during allocation + newSymbol = context.getSymbolAllocator().newSymbol(expression.getDisplayName(), origAggrReturnType, null); + newExpr = new CallExpression(aggFunctionSum, + newFunctionHandle, + expression.getType(), + ImmutableList.of(new VariableReferenceExpression(newSymbol.getName(), + origAggrReturnType))); + } + else { + FunctionHandle newFunctionHandle = metadata.getFunctionAndTypeManager().lookupFunction(expression.getDisplayName(), + TypeSignatureProvider.fromTypeSignatures(origAggrReturnTypeSig)); + + // newSymbol is going to point to origExpression, so using its returnType during allocation + newSymbol = context.getSymbolAllocator().newSymbol(expression.getDisplayName(), expression.getType(), null); + newExpr = new CallExpression(expression.getDisplayName(), + newFunctionHandle, + expression.getType(), + ImmutableList.of(new VariableReferenceExpression(newSymbol.getName(), + expression.getType()))); + } + context.getSymbolAllocator().updateSymbolType(symbol, newExpr.getType()); + + Aggregation newAggr = new Aggregation( + newExpr, + newExpr.getArguments(), + aggregation.isDistinct(), + aggregation.getFilter(), + aggregation.getOrderingScheme(), + aggregation.getMask()); + newAggregations.put(symbol, newAggr); + origAggrsNewRef.put(newSymbol, aggregation); + }); + return newAggregations.build(); + } + + private Map getAggregationsMatchingSymbols( + Map aggregations, Set symbols) + { + return aggregations.entrySet().stream() + .filter(entry -> { + List aggrSymbolList = SymbolsExtractor.extractAll(entry.getValue()); + return aggrSymbolList.size() == 1 && symbols.contains(aggrSymbolList.get(0)); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private Map getCommonAggregations(Map aggregations) + { + return aggregations.entrySet().stream() + .filter(entry -> { + List aggrSymbolList = SymbolsExtractor.extractAll(entry.getValue()); + return aggrSymbolList.size() == 0; + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private Set getJoinRequiredSymbols(JoinNode node) + { + return Streams.concat( + node.getCriteria().stream().map(JoinNode.EquiJoinClause::getLeft), + node.getCriteria().stream().map(JoinNode.EquiJoinClause::getRight), + node.getFilter().map(SymbolsExtractor::extractUnique).orElse(ImmutableSet.of()).stream(), + node.getLeftHashSymbol().map(ImmutableSet::of).orElse(ImmutableSet.of()).stream(), + node.getRightHashSymbol().map(ImmutableSet::of).orElse(ImmutableSet.of()).stream()) + .collect(toImmutableSet()); + } + + private List getPushedDownGroupingSet(AggregationNode aggregation, Set availableSymbols, + Set requiredJoinSymbols) + { + List groupingSet = aggregation.getGroupingKeys(); + + // keep symbols that are directly from the join's child (availableSymbols) + ImmutableList.Builder newGrpKeys = ImmutableList.builder(); + List pushedDownGroupingSet = groupingSet.stream() + .filter(availableSymbols::contains) + .collect(Collectors.toList()); + + // add missing required join symbols to grouping set + Set existingSymbols = new HashSet<>(pushedDownGroupingSet); + requiredJoinSymbols.stream() + .filter(existingSymbols::add) + .forEach(newGrpKeys::add); + // New Grp Keys - First have Join Keys and then Non-Join Grp Keys + newGrpKeys.addAll(pushedDownGroupingSet); + return newGrpKeys.build(); + } + + private boolean aggregationKeysSameAsJoinKeysOrSuperWithPresentInStart(List groupingKeys, List criteria, JoinNode joinNode) + { + // Check all join keys are part of grouping keys + // and Grouping keys which are not in Join keys, should belong to only one side + if (criteria.size() < 1) { + return false; + } + if (criteria.size() > groupingKeys.size()) { + return false; + } + Set grpKeySet = new HashSet<>(groupingKeys.subList(0, criteria.size())); + for (JoinNode.EquiJoinClause joinClause : criteria) { + if (grpKeySet.contains(joinClause.getLeft())) { + grpKeySet.remove(joinClause.getLeft()); + } + else if (grpKeySet.contains(joinClause.getRight())) { + grpKeySet.remove(joinClause.getRight()); + } + else { + return false; + } + } + if (criteria.size() == groupingKeys.size()) { + return true; + } + Set remainingGrpKeys = new HashSet<>(groupingKeys.subList(criteria.size(), groupingKeys.size())); + if (allSymbolsOn(remainingGrpKeys, joinNode.getLeft().getOutputSymbols())) { + return true; + } + else if (allSymbolsOn(remainingGrpKeys, joinNode.getRight().getOutputSymbols())) { + return true; + } + return false; + } + + private static boolean allSymbolsOn(Set grpKeySet, List symbols) + { + return new HashSet<>(symbols).containsAll(grpKeySet); + } + + private boolean isJoinKeysSubsetOfAggrGroupKeys(List groupingKeys, List criteria) + { + // Check all join keys are part of grouping keys(Any Place) + // and Grouping keys which are not in Join keys can belong to any side + if (criteria.size() < 1) { + return false; + } + if (criteria.size() > groupingKeys.size()) { + return false; + } + Set grpKeySet = new HashSet<>(groupingKeys); + for (JoinNode.EquiJoinClause joinClause : criteria) { + if (grpKeySet.contains(joinClause.getLeft())) { + grpKeySet.remove(joinClause.getLeft()); + } + else if (grpKeySet.contains(joinClause.getRight())) { + grpKeySet.remove(joinClause.getRight()); + } + else { + return false; + } + } + // Exact match between group by and Join + if (criteria.size() == groupingKeys.size()) { + return true; + } + return true; + } + + public static enum GroupJoinAggregationFunction + { + SUM("sum"), + COUNT("count"), + AVG("avg"), + MIN("min"), + MAX("max"), + STDDEV_SAMP("stddev_samp"), + STDDEV("stddev"); + + private final String name; + + public static final Set SUPPORTED_FUNCTIONS = Stream.of(GroupJoinAggregationFunction.values()).map(GroupJoinAggregationFunction::getName).collect(Collectors.toSet()); + + GroupJoinAggregationFunction(String name) + { + this.name = name; + } + + public String getName() + { + return this.name; + } + + public String toString() + { + return this.name; + } + } +} diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/MergePartialAggregationWithJoinPushProject.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/MergePartialAggregationWithJoinPushProject.java new file mode 100644 index 0000000000000000000000000000000000000000..3efe09790331febe691432252c2a520df435bebb --- /dev/null +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/MergePartialAggregationWithJoinPushProject.java @@ -0,0 +1,147 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.sql.planner.iterative.rule; + +import io.prestosql.matching.Capture; +import io.prestosql.matching.Captures; +import io.prestosql.matching.Pattern; +import io.prestosql.metadata.Metadata; +import io.prestosql.spi.plan.AggregationNode; +import io.prestosql.spi.plan.Assignments; +import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.PlanNode; +import io.prestosql.spi.plan.ProjectNode; +import io.prestosql.spi.plan.Symbol; +import io.prestosql.spi.relation.RowExpression; +import io.prestosql.spi.relation.VariableReferenceExpression; +import io.prestosql.sql.planner.SymbolsExtractor; +import io.prestosql.sql.planner.TypeProvider; + +import java.util.HashSet; +import java.util.List; +import java.util.Map.Entry; + +import static io.prestosql.sql.planner.plan.Patterns.aggregation; +import static io.prestosql.sql.planner.plan.Patterns.join; +import static io.prestosql.sql.planner.plan.Patterns.project; +import static io.prestosql.sql.planner.plan.Patterns.source; + +public class MergePartialAggregationWithJoinPushProject + extends MergePartialAggregationWithJoin +{ + private static final Capture PROJECT_NODE = Capture.newCapture(); + + private static final Pattern PATTERN = aggregation() + .matching(MergePartialAggregationWithJoin::isSupportedAggregationNode) + .with(source().matching(project().capturedAs(PROJECT_NODE) + .with(source().matching(join().capturedAs(JOIN_NODE))))); + + public MergePartialAggregationWithJoinPushProject(Metadata metadata) + { + super(metadata); + } + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(AggregationNode node, Captures captures, Context context) + { + ProjectNode projectNode = captures.get(PROJECT_NODE); + JoinNode joinNode = captures.get(JOIN_NODE); + if (!nonTrivialProjection(projectNode)) { + return Result.empty(); + } + if (joinNode.getType() != JoinNode.Type.INNER) { + return Result.empty(); + } + + // Check if Project can be pushed down through join + // Check if aggregations can be pushed down through join + // First push Project through Join + // Then call super.apply() or equivalent function to apply the rule + Assignments assignments = projectNode.getAssignments(); + Assignments.Builder leftAssignments = Assignments.builder(); + Assignments.Builder rightAssignments = Assignments.builder(); + HashSet leftSymbolSet = new HashSet<>(joinNode.getLeft().getOutputSymbols()); + HashSet rightSymbolSet = new HashSet<>(joinNode.getRight().getOutputSymbols()); + for (Entry assignment : assignments.entrySet()) { + List symbols = SymbolsExtractor.extractAll(assignment.getValue()); + if (leftSymbolSet.containsAll(symbols)) { + leftAssignments.put(assignment.getKey(), assignment.getValue()); + } + else if (rightSymbolSet.containsAll(symbols)) { + rightAssignments.put(assignment.getKey(), assignment.getValue()); + } + else { + return Result.empty(); + } + } + TypeProvider typeProvider = context.getSymbolAllocator().getTypes(); + for (Entry df : joinNode.getDynamicFilters().entrySet()) { + if (leftSymbolSet.contains(df.getValue())) { + leftAssignments.put(df.getValue(), new VariableReferenceExpression(df.getValue().getName(), typeProvider.get(df.getValue()))); + } + else if (rightSymbolSet.contains(df.getValue())) { + rightAssignments.put(df.getValue(), new VariableReferenceExpression(df.getValue().getName(), typeProvider.get(df.getValue()))); + } + } + + PlanNode leftNode = joinNode.getLeft(); + Assignments build = leftAssignments.build(); + if (build.size() > 0) { + leftNode = new ProjectNode(context.getIdAllocator().getNextId(), joinNode.getLeft(), build); + } + + PlanNode rightNode = joinNode.getRight(); + build = rightAssignments.build(); + if (build.size() > 0) { + rightNode = new ProjectNode(context.getIdAllocator().getNextId(), joinNode.getRight(), build); + } + joinNode = new JoinNode(joinNode.getId(), + joinNode.getType(), + leftNode, + rightNode, + joinNode.getCriteria(), + projectNode.getOutputSymbols(), + joinNode.getFilter(), + joinNode.getLeftHashSymbol(), + joinNode.getRightHashSymbol(), + joinNode.getDistributionType(), + joinNode.isSpillable(), + joinNode.getDynamicFilters()); + AggregationNode nodeNew = new AggregationNode(node.getId(), + joinNode, + node.getAggregations(), + node.getGroupingSets(), + node.getPreGroupedSymbols(), + node.getStep(), + node.getHashSymbol(), + node.getGroupIdSymbol(), + node.getAggregationType(), + node.getFinalizeSymbol()); + return checkAndApplyRule(nodeNew, context, joinNode); + } + + private static boolean nonTrivialProjection(ProjectNode project) + { + return !project.getAssignments() + .getExpressions().stream() + .allMatch(expression -> expression instanceof VariableReferenceExpression); + } +} diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TablePushdown.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TablePushdown.java index cfd1fb773bc578b6ec6821c968043b793d7c4da8..f7f7a35b1d178f03bf9ab8618ad1470b8dfd6f8c 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TablePushdown.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TablePushdown.java @@ -27,6 +27,7 @@ import io.prestosql.spi.metadata.TableHandle; import io.prestosql.spi.plan.AggregationNode; import io.prestosql.spi.plan.Assignments; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.ProjectNode; import io.prestosql.spi.plan.Symbol; @@ -187,12 +188,12 @@ public class TablePushdown * */ stack.push(new NodeWithTreeDirection(node, DIRECTION.LEFT)); if (!(node instanceof TableScanNode)) { - if (node instanceof JoinNode) { + if (node instanceof JoinNode || node instanceof JoinOnAggregationNode) { if (updateStack(resolveNodeFromGroupReference(node, 0, lookup), lookup, stack)) { return true; } else { - while (!(stack.peek().getNode() instanceof JoinNode)) { + while (!(stack.peek().getNode() instanceof JoinNode || stack.peek().getNode() instanceof JoinOnAggregationNode)) { stack.pop(); } NodeWithTreeDirection tempNode = stack.pop(); @@ -373,7 +374,7 @@ public class TablePushdown for (NodeWithTreeDirection nodeInPath : stack) { PlanNode node = nodeInPath.getNode(); - if (node instanceof JoinNode) { + if (node instanceof JoinNode || node instanceof JoinOnAggregationNode) { hasJoinInPath = true; break; } @@ -444,7 +445,7 @@ public class TablePushdown Stack intermediateOuterTableStack = new Stack<>(); // First pop the stack till we reach a join node and push into another intermediateOuterTableStack. - while (!(stack.peek().getNode() instanceof JoinNode)) { + while (!(stack.peek().getNode() instanceof JoinNode || stack.peek().getNode() instanceof JoinOnAggregationNode)) { intermediateOuterTableStack.push(stack.pop()); } diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TransformUnCorrelatedSubquerySelfJoinToWindowFunctions.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TransformUnCorrelatedSubquerySelfJoinToWindowFunctions.java index 6653d0d6c603e6a6408507bc54961ca10045014d..f4ac8af795029227314d3bdb26981ee5e37bb4af 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TransformUnCorrelatedSubquerySelfJoinToWindowFunctions.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TransformUnCorrelatedSubquerySelfJoinToWindowFunctions.java @@ -30,6 +30,7 @@ import io.prestosql.spi.plan.Assignments; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupReference; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.OrderingScheme; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.ProjectNode; @@ -616,6 +617,13 @@ public class TransformUnCorrelatedSubquerySelfJoinToWindowFunctions return super.visitJoin(node, context); } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Details context) + { + context.joinClauses.addAll(node.getCriteria()); + return super.visitJoinOnAggregation(node, context); + } + @Override public Void visitProject(ProjectNode node, Details context) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TransformUncorrelatedSubquerySelfJoinAggregatesToWindowFunction.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TransformUncorrelatedSubquerySelfJoinAggregatesToWindowFunction.java index e81c262105becfe8ed716dfd7f2c60cc449b234c..d6d6fd4461d03b3199a8d88437505e7dd72d9b99 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TransformUncorrelatedSubquerySelfJoinAggregatesToWindowFunction.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/TransformUncorrelatedSubquerySelfJoinAggregatesToWindowFunction.java @@ -30,6 +30,7 @@ import io.prestosql.spi.plan.Assignments; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupReference; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.OrderingScheme; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.ProjectNode; @@ -590,6 +591,13 @@ public class TransformUncorrelatedSubquerySelfJoinAggregatesToWindowFunction return super.visitJoin(node, context); } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Details context) + { + context.joinClauses.addAll(node.getCriteria()); + return super.visitJoinOnAggregation(node, context); + } + @Override public Void visitProject(ProjectNode node, Details context) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddLocalExchanges.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddLocalExchanges.java index 033135b12115d3b0031400a6b4f08ebc6b38d5d3..b01faba719b65da5f4365a3fc63876bae3d81086 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddLocalExchanges.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddLocalExchanges.java @@ -26,6 +26,7 @@ import io.prestosql.spi.connector.SortingProperty; import io.prestosql.spi.plan.AggregationNode; import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; @@ -706,6 +707,41 @@ public class AddLocalExchanges return rebaseAndDeriveProperties(node, ImmutableList.of(probe, build)); } + @Override + public PlanWithProperties visitJoinOnAggregation(JoinOnAggregationNode inputNode, StreamPreferredProperties parentPreferences) + { + JoinOnAggregationNode node = inputNode; + PlanWithProperties probe = planAndEnforce( + node.getLeft(), + defaultParallelism(session), + parentPreferences.constrainTo(node.getLeft().getOutputSymbols()).withDefaultParallelism(session)); + + if (isSpillEnabled(session)) { + if (probe.getProperties().getDistribution() != FIXED) { + // Disable spill for joins over non-fixed streams as otherwise we would need to insert local exchange. + // Such local exchanges can hurt performance when spill is not triggered. + // When spill is not triggered it should not induce performance penalty. + node = node.withSpillable(false); + } + else { + node = node.withSpillable(true); + } + } + + // this build consumes the input completely, so we do not pass through parent preferences + List buildHashSymbols = Lists.transform(node.getCriteria(), JoinNode.EquiJoinClause::getRight); + StreamPreferredProperties buildPreference; + if (getTaskConcurrency(session) > 1) { + buildPreference = exactlyPartitionedOn(buildHashSymbols); + } + else { + buildPreference = singleStream(); + } + PlanWithProperties build = planAndEnforce(node.getRight(), buildPreference, buildPreference); + + return rebaseAndDeriveProperties(node, ImmutableList.of(probe, build)); + } + @Override public PlanWithProperties visitSemiJoin(SemiJoinNode node, StreamPreferredProperties parentPreferences) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddReuseExchange.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddReuseExchange.java index 97caae6c04fbd678dcd31a280616b157cbac132d..418e6516d4fd09398ce760f7832d35475f06eb83 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddReuseExchange.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddReuseExchange.java @@ -24,6 +24,7 @@ import io.prestosql.spi.plan.Assignments; import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeIdAllocator; import io.prestosql.spi.plan.ProjectNode; @@ -202,6 +203,25 @@ public class AddReuseExchange return node; } + @Override + public PlanNode visitJoinOnAggregation(JoinOnAggregationNode inputNode, RewriteContext context) + { + JoinOnAggregationNode node = inputNode; + node = (JoinOnAggregationNode) visitPlan(node, context); + // verify right side + TableScanNode left = (TableScanNode) getTableScanNode(node.getLeft()); + TableScanNode right = (TableScanNode) getTableScanNode(node.getRight()); + if (left != null && right != null && WrapperScanNode.of(left).equals(WrapperScanNode.of(right))) { + WrapperScanNode leftNode = WrapperScanNode.of(left); + if (planNodeListHashMap.get(leftNode) != null) { + // These nodes are part of reuse exchange, adjust them. + planNodeListHashMap.remove(leftNode); + } + } + + return node; + } + private double getMaxTableSizeToEnableReuseExchange(TableStatistics stats, Map assignments) { return assignments.values().stream().map(x -> stats.getColumnStatistics().get(x)).filter(x -> x != null) diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java index 8f4445167a81cad3c693f1098dd8d50f3acf3c24..dab3562c60960157fa95bc2eb3e7d754701837a0 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java @@ -22,6 +22,7 @@ import io.prestosql.spi.metadata.TableHandle; import io.prestosql.spi.operator.ReuseExchangeOperator; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeIdAllocator; import io.prestosql.spi.plan.ProjectNode; @@ -393,6 +394,12 @@ public class BeginTableWrite return locateTableScanHandle(joinNode.getLeft()); } } + if (node instanceof JoinOnAggregationNode) { + JoinOnAggregationNode joinNode = (JoinOnAggregationNode) node; + if (joinNode.getType() == JoinNode.Type.INNER) { + return locateTableScanHandle(joinNode.getLeft()); + } + } throw new IllegalArgumentException("Invalid descendant for DeleteNode or UpdateNode" + node.getClass().getName()); } @@ -458,6 +465,13 @@ public class BeginTableWrite return replaceChildren(node, ImmutableList.of(source, joinNode.getRight())); } } + if (node instanceof JoinOnAggregationNode) { + JoinOnAggregationNode joinNode = (JoinOnAggregationNode) node; + if (joinNode.getType() == JoinNode.Type.INNER) { + PlanNode source = rewriteModifyTableScan(joinNode.getLeft(), handle); + return replaceChildren(node, ImmutableList.of(source, joinNode.getRight())); + } + } throw new IllegalArgumentException("Invalid descendant for DeleteNode or UpdateNode: " + node.getClass().getName()); } } diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/HashGenerationOptimizer.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/HashGenerationOptimizer.java index 5cd7a336e47cbd6e9c28c50087a040e0ca28e6f5..3db2392638728927189b5780fed8f8dbb6a5505b 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/HashGenerationOptimizer.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/HashGenerationOptimizer.java @@ -14,6 +14,7 @@ package io.prestosql.sql.planner.optimizations; import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; @@ -36,6 +37,8 @@ import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; +import io.prestosql.spi.plan.JoinOnAggregationNode.JoinInternalAggregation; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeIdAllocator; @@ -53,6 +56,7 @@ import io.prestosql.sql.planner.FunctionCallBuilder; import io.prestosql.sql.planner.Partitioning.ArgumentBinding; import io.prestosql.sql.planner.PartitioningScheme; import io.prestosql.sql.planner.PlanSymbolAllocator; +import io.prestosql.sql.planner.SymbolsExtractor; import io.prestosql.sql.planner.TypeProvider; import io.prestosql.sql.planner.plan.ApplyNode; import io.prestosql.sql.planner.plan.DistinctLimitNode; @@ -97,6 +101,7 @@ import static io.prestosql.spi.connector.CatalogSchemaName.DEFAULT_NAMESPACE; import static io.prestosql.spi.function.FunctionKind.SCALAR; import static io.prestosql.spi.function.Signature.mangleOperatorName; import static io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT; +import static io.prestosql.spi.plan.AggregationNode.singleGroupingSet; import static io.prestosql.spi.plan.JoinNode.Type.INNER; import static io.prestosql.spi.plan.JoinNode.Type.LEFT; import static io.prestosql.spi.plan.JoinNode.Type.RIGHT; @@ -411,6 +416,197 @@ public class HashGenerationOptimizer hashSymbolsWithParentPreferences); } + @Override + public PlanWithProperties visitJoinOnAggregation(JoinOnAggregationNode node, HashComputationSet parentPreference) + { + List clauses = node.getCriteria(); + + // join does not pass through preferred hash symbols since they take more memory and since + // the join node filters, may take more compute + Optional leftHashComputation = computeHash(metadata, planSymbolAllocator, Lists.transform(clauses, JoinNode.EquiJoinClause::getLeft)); + Optional rightHashComputation = computeHash(metadata, planSymbolAllocator, Lists.transform(clauses, JoinNode.EquiJoinClause::getRight)); + + // build map of all hash symbols + // NOTE: Full outer join doesn't use hash symbols + BiMap allHashSymbols = HashBiMap.create(); + PlanWithProperties left; + PlanWithProperties right; + Optional leftHashSymbol; + Optional rightHashSymbol; + + Optional leftHashSymbolAggr; + Optional rightHashSymbolAggr; + + List leftGrpByKeys = node.getLeftAggr().getGroupingKeys(); + List rightGrpByKeys = node.getRightAggr().getGroupingKeys(); + List aggrOnLeftGrpByKeys = node.getAggrOnLeft().getGroupingKeys(); + List aggrOnRightGrpByKeys = node.getAggrOnRight().getGroupingKeys(); + + // Update Hash Symbol in Left and Right Aggregation of Join + // If Join Keys and Grouping Keys are same, then only join hash symbol can be re-used, otherwise need to calculate for grouping keys + if (!canJoinHashSymbolBeUsedForJoinInternalAggregation(Lists.transform(clauses, JoinNode.EquiJoinClause::getLeft), leftGrpByKeys)) { + left = planAndEnforce(node.getLeft(), new HashComputationSet(leftHashComputation), true, new HashComputationSet(leftHashComputation)); + leftHashSymbol = Optional.of(left.getRequiredHashSymbol(leftHashComputation.get())); + + List groupingKeys = node.getLeftAggr().getGroupingKeys(); + leftGrpByKeys = ImmutableList.builder() + .addAll(groupingKeys.subList(0, node.getCriteria().size())) + .add(leftHashSymbol.get()) + .addAll(groupingKeys.subList(node.getCriteria().size(), groupingKeys.size())) + .build(); + + groupingKeys = node.getAggrOnLeft().getGroupingKeys(); + aggrOnLeftGrpByKeys = ImmutableList.builder() + .addAll(groupingKeys.subList(0, node.getCriteria().size())) + .add(leftHashSymbol.get()) + .addAll(groupingKeys.subList(node.getCriteria().size(), groupingKeys.size())) + .build(); + + Optional leftAggrHashComputation = computeHash(metadata, planSymbolAllocator, leftGrpByKeys); + ImmutableSetMultimap.Builder builder = ImmutableSetMultimap.builder(); + builder.put(leftAggrHashComputation.get(), leftAggrHashComputation.get()); + left = planAndEnforce(left.getNode(), new HashComputationSet(builder.build()), true, new HashComputationSet(builder.build())); + leftHashSymbolAggr = Optional.ofNullable(left.getRequiredHashSymbol(leftAggrHashComputation.get())); + } + else { + left = planAndEnforce(node.getLeft(), new HashComputationSet(leftHashComputation), true, new HashComputationSet(leftHashComputation)); + leftHashSymbol = Optional.of(left.getRequiredHashSymbol(leftHashComputation.get())); + leftHashSymbolAggr = leftHashSymbol; + } + + if (!canJoinHashSymbolBeUsedForJoinInternalAggregation(Lists.transform(clauses, JoinNode.EquiJoinClause::getRight), rightGrpByKeys)) { + right = planAndEnforce(node.getRight(), new HashComputationSet(rightHashComputation), true, new HashComputationSet(rightHashComputation)); + rightHashSymbol = Optional.of(right.getRequiredHashSymbol(rightHashComputation.get())); + + List groupingKeys = node.getRightAggr().getGroupingKeys(); + rightGrpByKeys = ImmutableList.builder() + .addAll(groupingKeys.subList(0, node.getCriteria().size())) + .add(rightHashSymbol.get()) + .addAll(groupingKeys.subList(node.getCriteria().size(), groupingKeys.size())) + .build(); + + groupingKeys = node.getAggrOnRight().getGroupingKeys(); + aggrOnRightGrpByKeys = ImmutableList.builder() + .addAll(groupingKeys.subList(0, node.getCriteria().size())) + .add(rightHashSymbol.get()) + .addAll(groupingKeys.subList(node.getCriteria().size(), groupingKeys.size())) + .build(); + + Optional rightAggrHashComputation = computeHash(metadata, planSymbolAllocator, rightGrpByKeys); + ImmutableSetMultimap.Builder builder = ImmutableSetMultimap.builder(); + builder.put(rightAggrHashComputation.get(), rightAggrHashComputation.get()); + right = planAndEnforce(right.getNode(), new HashComputationSet(builder.build()), true, new HashComputationSet(builder.build())); + rightHashSymbolAggr = Optional.ofNullable(right.getRequiredHashSymbol(rightAggrHashComputation.get())); + } + else { + right = planAndEnforce(node.getRight(), new HashComputationSet(rightHashComputation), true, new HashComputationSet(rightHashComputation)); + rightHashSymbol = Optional.of(right.getRequiredHashSymbol(rightHashComputation.get())); + rightHashSymbolAggr = rightHashSymbol; + } + + if (node.getType() == INNER || node.getType() == LEFT) { + allHashSymbols.putAll(left.getHashSymbols()); + } + if (node.getType() == INNER || node.getType() == RIGHT) { + allHashSymbols.putAll(right.getHashSymbols()); + } + + // retain only hash symbols preferred by parent nodes + Map hashSymbolsWithParentPreferences = + allHashSymbols.entrySet() + .stream() + .filter(entry -> parentPreference.getHashes().contains(entry.getKey())) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)); + + JoinInternalAggregation leftAggr = new JoinInternalAggregation(node.getLeftAggr().getId(), + left.getNode(), + node.getLeftAggr().getAggregations(), + singleGroupingSet(leftGrpByKeys), + ImmutableList.of(), + node.getLeftAggr().getStep(), + leftHashSymbolAggr, + node.getLeftAggr().getGroupIdSymbol(), + node.getLeftAggr().getAggregationType(), + node.getLeftAggr().getFinalizeSymbol()); + + JoinInternalAggregation rightAggr = new JoinInternalAggregation(node.getRightAggr().getId(), + right.getNode(), + node.getRightAggr().getAggregations(), + singleGroupingSet(rightGrpByKeys), + ImmutableList.of(), + node.getRightAggr().getStep(), + rightHashSymbolAggr, + node.getRightAggr().getGroupIdSymbol(), + node.getRightAggr().getAggregationType(), + node.getRightAggr().getFinalizeSymbol()); + + JoinInternalAggregation aggrOnLeftAggr = new JoinInternalAggregation(node.getAggrOnLeft().getId(), + leftAggr, + node.getAggrOnLeft().getAggregations(), + singleGroupingSet(aggrOnLeftGrpByKeys), + ImmutableList.of(), + node.getAggrOnLeft().getStep(), + leftHashSymbolAggr, + node.getAggrOnLeft().getGroupIdSymbol(), + node.getAggrOnLeft().getAggregationType(), + node.getAggrOnLeft().getFinalizeSymbol()); + + JoinInternalAggregation aggrOnRightAggr = new JoinInternalAggregation(node.getAggrOnRight().getId(), + rightAggr, + node.getAggrOnRight().getAggregations(), + singleGroupingSet(aggrOnRightGrpByKeys), + ImmutableList.of(), + node.getAggrOnRight().getStep(), + rightHashSymbolAggr, + node.getAggrOnRight().getGroupIdSymbol(), + node.getAggrOnRight().getAggregationType(), + node.getAggrOnRight().getFinalizeSymbol()); + + ImmutableList.Builder newOutputSymbolBuilder = ImmutableList.builder(); + newOutputSymbolBuilder + .addAll(aggrOnLeftAggr.getOutputSymbols()) + .addAll(aggrOnRightAggr.getOutputSymbols()); + return new PlanWithProperties( + new JoinOnAggregationNode( + node.getId(), + node.getType(), + node.getCriteria(), + node.getFilter(), + leftHashSymbol, + rightHashSymbol, + node.getDistributionType(), + node.isSpillable(), + node.getDynamicFilters(), + leftAggr, + rightAggr, + aggrOnLeftAggr, + aggrOnRightAggr, + newOutputSymbolBuilder.build()), + hashSymbolsWithParentPreferences); + } + + private boolean canJoinHashSymbolBeUsedForJoinInternalAggregation(List joinSymbols, + List groupingKeys) + { + if (joinSymbols.size() == groupingKeys.size() && joinSymbols.containsAll(groupingKeys)) { + return true; + } + return false; + } + + private boolean canHashSymbolUseForJoinInternalAggregation(Optional hashSymbol, + List groupingKeys, + BiMap allHashSymbols) + { + if (hashSymbol.isPresent()) { + HashComputation hashComputation = allHashSymbols.inverse().get(hashSymbol.get()); + if (hashComputation.getFields().size() == groupingKeys.size() && hashComputation.getFields().containsAll(groupingKeys)) { + return true; + } + } + return false; + } + @Override public PlanWithProperties visitSemiJoin(SemiJoinNode node, HashComputationSet parentPreference) { @@ -685,29 +881,60 @@ public class HashGenerationOptimizer // create a new project node with all assignments from the original node Assignments.Builder newAssignments = Assignments.builder(); - newAssignments.putAll(node.getAssignments()); + Assignments assignments = node.getAssignments(); + Map hashSymbolMap = new HashMap<>(); + for (Symbol symbol : assignments.getOutputs()) { + if (symbol.getName().startsWith("$hashvalue")) { + HashComputationSet hashComputationSet = new HashComputationSet(Optional.of(new HashComputation(metadata, planSymbolAllocator, SymbolsExtractor.extractAll(assignments.get(symbol))))); + hashSymbolMap.put(hashComputationSet, symbol); + newAssignments.put(symbol, assignments.get(symbol)); + } + else { + newAssignments.put(symbol, assignments.get(symbol)); + } + } // and all hash symbols that could be translated to the source symbols Map allHashSymbols = new HashMap<>(); for (HashComputation hashComputation : sourceContext.getHashes()) { - Symbol hashSymbol = child.getHashSymbols().get(hashComputation); - RowExpression hashExpression; - if (hashSymbol == null) { - hashSymbol = planSymbolAllocator.newHashSymbol(); - hashExpression = hashComputation.getHashExpression(); + List fields = hashComputation.getFields(); + Symbol symbol = canComputeWith(fields, hashSymbolMap); + if (symbol == null) { + Symbol hashSymbol = child.getHashSymbols().get(hashComputation); + RowExpression hashExpression; + if (hashSymbol == null) { + hashSymbol = planSymbolAllocator.newHashSymbol(); + hashExpression = hashComputation.getHashExpression(); + } + else { + hashExpression = toVariableReference(hashSymbol, planSymbolAllocator.getTypes().get(hashSymbol)); + } + newAssignments.put(hashSymbol, hashExpression); + for (HashComputation sourceHashComputation : sourceContext.lookup(hashComputation)) { + allHashSymbols.put(sourceHashComputation, hashSymbol); + } } else { - hashExpression = toVariableReference(hashSymbol, planSymbolAllocator.getTypes().get(hashSymbol)); - } - newAssignments.put(hashSymbol, hashExpression); - for (HashComputation sourceHashComputation : sourceContext.lookup(hashComputation)) { - allHashSymbols.put(sourceHashComputation, hashSymbol); + for (HashComputation sourceHashComputation : sourceContext.lookup(hashComputation)) { + allHashSymbols.put(sourceHashComputation, symbol); + } } } return new PlanWithProperties(new ProjectNode(node.getId(), child.getNode(), newAssignments.build()), allHashSymbols); } + private Symbol canComputeWith(List fields, Map hashSymbolMap) + { + for (Map.Entry entry : hashSymbolMap.entrySet()) { + boolean anyMatch = entry.getKey().getHashes().stream().anyMatch(hashComputation -> fields.size() == hashComputation.getFields().size() && fields.containsAll(hashComputation.getFields())); + if (anyMatch) { + return entry.getValue(); + } + } + return null; + } + @Override public PlanWithProperties visitUnnest(UnnestNode node, HashComputationSet parentPreference) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PredicatePushDown.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PredicatePushDown.java index 39ef407cc4e55491c96a6c81989d90e361f131d5..e6eb95966854b7bb9b6349ae34becd862cd3ac97 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PredicatePushDown.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PredicatePushDown.java @@ -40,6 +40,7 @@ import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeIdAllocator; @@ -1248,7 +1249,8 @@ public class PredicatePushDown // See if we can push the left effective predicate to the right side for (RowExpression conjunct : new RowExpressionEqualityInference.Builder(metadata, typeManager).nonInferrableConjuncts(leftEffectivePredicate)) { // do not push down dynamic filters here - if (!(node instanceof JoinNode && getDescriptor(conjunct).isPresent() && ((JoinNode) node).getDynamicFilters().keySet().contains(getDescriptor(conjunct).get().getId()))) { + if (!((node instanceof JoinNode && getDescriptor(conjunct).isPresent() && ((JoinNode) node).getDynamicFilters().keySet().contains(getDescriptor(conjunct).get().getId())) + || (node instanceof JoinOnAggregationNode && getDescriptor(conjunct).isPresent() && ((JoinOnAggregationNode) node).getDynamicFilters().keySet().contains(getDescriptor(conjunct).get().getId())))) { RowExpression rewritten = allInference.rewriteExpression(conjunct, not(in(leftVariables))); if (rewritten != null) { rightPushDownConjuncts.add(rewritten); diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PropertyDerivations.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PropertyDerivations.java index 76254d322d6622b29e25a23d5cbb5d5c091a410e..3f5ca4bb91eb0933bbf05252f28ab0280829565f 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PropertyDerivations.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PropertyDerivations.java @@ -34,6 +34,7 @@ import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.OrderingScheme; @@ -514,6 +515,32 @@ public class PropertyDerivations } } + @Override + public ActualProperties visitJoinOnAggregation(JoinOnAggregationNode node, List inputProperties) + { + ActualProperties probeProperties = inputProperties.get(0); + ActualProperties buildProperties = inputProperties.get(1); + + boolean unordered = spillPossible(session, node.getType()); + + switch (node.getType()) { + case INNER: + probeProperties = probeProperties.translate(column -> filterOrRewrite(node.getOutputSymbols(), node.getCriteria(), column)); + buildProperties = buildProperties.translate(column -> filterOrRewrite(node.getOutputSymbols(), node.getCriteria(), column)); + + Map constants = new HashMap<>(); + constants.putAll(probeProperties.getConstants()); + constants.putAll(buildProperties.getConstants()); + + return ActualProperties.builderFrom(probeProperties) + .constants(constants) + .unordered(unordered) + .build(); + default: + throw new UnsupportedOperationException("Unsupported group join type: " + node.getType()); + } + } + @Override public ActualProperties visitSemiJoin(SemiJoinNode node, List inputProperties) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PruneCTENodes.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PruneCTENodes.java index 3cce8552ae88ea7ceec0c6dcd0eb30ed2af5bb76..8b3537138560a0489adc2864f750f0f69c375e12 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PruneCTENodes.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PruneCTENodes.java @@ -22,6 +22,7 @@ import io.prestosql.spi.plan.AggregationNode; import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeId; import io.prestosql.spi.plan.PlanNodeIdAllocator; @@ -148,6 +149,35 @@ public class PruneCTENodes return true; } + private boolean checkCTELevel(JoinOnAggregationNode node) + { + int leftLevel = getChildCTELevel(node.getLeft(), 0); + int rightLevel = getChildCTELevel(node.getRight(), 0); + if (leftLevel == 0 || rightLevel == 0) { + return true; + } + else if (leftLevel == rightLevel) { + return false; + } + return true; + } + + @Override + public PlanNode visitJoinOnAggregation(JoinOnAggregationNode node, RewriteContext context) + { + Integer left = getChildCTERefNum(node.getLeft()); + Integer right = getChildCTERefNum(node.getRight()); + if (left != null && right != null && left.equals(right) && checkCTELevel(node)) { + if (!isNodeAlreadyVisited) { + PlanNodeId probeCteNodeId = getProbeCTENodeId(node.getLeft()); + if (probeCteNodeId != null) { + probeCTEToPrune.add(probeCteNodeId); + } + } + } + return context.defaultRewrite(node, context.get()); + } + private PlanNodeId getProbeCTENodeId(PlanNode node) { if (node instanceof CTEScanNode) { @@ -168,6 +198,9 @@ public class PruneCTENodes else if (node instanceof JoinNode) { return getProbeCTENodeId(((JoinNode) node).getLeft()); } + else if (node instanceof JoinOnAggregationNode) { + return getProbeCTENodeId(((JoinOnAggregationNode) node).getLeft()); + } return null; } @@ -210,6 +243,10 @@ public class PruneCTENodes PlanNode joinNode = ((JoinNode) node).getLeft(); return getChildCTERefNum(joinNode); } + else if (node instanceof JoinOnAggregationNode) { + PlanNode joinNode = ((JoinOnAggregationNode) node).getLeft(); + return getChildCTERefNum(joinNode); + } return null; } @@ -237,6 +274,11 @@ public class PruneCTENodes level++; return getChildCTELevel(joinNode, level); } + else if (node instanceof JoinOnAggregationNode) { + PlanNode joinNode = ((JoinOnAggregationNode) node).getLeft(); + level++; + return getChildCTELevel(joinNode, level); + } return level; } @@ -265,6 +307,18 @@ public class PruneCTENodes return node.getSource(); } } + + // If there is a self join below CTE node, then CTE should be removed. + if (node.getSource() instanceof JoinOnAggregationNode) { + // check if this join is self join + TableHandle left = getTableHandle(((JoinOnAggregationNode) node.getSource()).getLeft()); + TableHandle right = getTableHandle(((JoinOnAggregationNode) node.getSource()).getRight()); + if (left != null && right != null && left.getConnectorHandle().equals(right.getConnectorHandle())) { + // both tables are same, means it is self join. + node = (CTEScanNode) visitPlan(node, context); + return node.getSource(); + } + } } if (!isNodeAlreadyVisited) { cteUsageMap.merge(commonCTERefNum, 1, Integer::sum); diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/StreamPropertyDerivations.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/StreamPropertyDerivations.java index 5ee06e922171c88167fbf0909134fd1f353d9aa3..2338b2491df2a95de4406fade881d12e695f1cc4 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/StreamPropertyDerivations.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/StreamPropertyDerivations.java @@ -28,6 +28,7 @@ import io.prestosql.spi.plan.CTEScanNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; @@ -226,6 +227,27 @@ public final class StreamPropertyDerivations } } + @Override + public StreamProperties visitJoinOnAggregation(JoinOnAggregationNode node, List inputProperties) + { + StreamProperties leftProperties = inputProperties.get(0); + boolean unordered = spillPossible(session, node); + + switch (node.getType()) { + case INNER: + return leftProperties + .translate(column -> PropertyDerivations.filterOrRewrite(node.getOutputSymbols(), node.getCriteria(), column)) + .unordered(unordered); + default: + throw new UnsupportedOperationException("Unsupported group join type: " + node.getType()); + } + } + + private static boolean spillPossible(Session session, JoinOnAggregationNode node) + { + return isSpillEnabled(session) && node.isSpillable().orElseThrow(() -> new IllegalArgumentException("spillable not yet set")); + } + private static boolean spillPossible(Session session, JoinNode node) { return isSpillEnabled(session) && node.isSpillable().orElseThrow(() -> new IllegalArgumentException("spillable not yet set")); diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/joins/JoinGraph.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/joins/JoinGraph.java index cecb10ce7d614848ea8ffb065029cfb785b7484a..f956762559eb06b11cb2f06dcde384085c27e104 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/joins/JoinGraph.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/joins/JoinGraph.java @@ -19,6 +19,7 @@ import com.google.common.collect.Multimap; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupReference; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.PlanNodeId; import io.prestosql.spi.plan.ProjectNode; @@ -274,6 +275,25 @@ public class JoinGraph return graph; } + @Override + public JoinGraph visitJoinOnAggregation(JoinOnAggregationNode node, Context context) + { + //TODO: add support for non inner joins + if (node.getType() != INNER) { + return visitPlan(node, context); + } + + JoinGraph left = node.getLeft().accept(this, context); + JoinGraph right = node.getRight().accept(this, context); + + JoinGraph graph = left.joinWith(right, node.getCriteria(), context, node.getId()); + + if (node.getFilter().isPresent()) { + return graph.withFilter(node.getFilter().get()); + } + return graph; + } + @Override public JoinGraph visitProject(ProjectNode node, Context context) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/plan/Patterns.java b/presto-main/src/main/java/io/prestosql/sql/planner/plan/Patterns.java index 3a796f2afb18784858db23fadcf1575871babc63..3363af61bfbbeb8390a4d5257d4836155ede4242 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/plan/Patterns.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/plan/Patterns.java @@ -22,6 +22,7 @@ import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.IntersectNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; @@ -114,6 +115,11 @@ public class Patterns return typeOf(JoinNode.class); } + public static Pattern joinOnAggregation() + { + return typeOf(JoinOnAggregationNode.class); + } + public static Pattern spatialJoin() { return typeOf(SpatialJoinNode.class); diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/planprinter/PlanPrinter.java b/presto-main/src/main/java/io/prestosql/sql/planner/planprinter/PlanPrinter.java index f8edb93f847b911e0f7796a36c24afb221429dc4..d4024b85b3f054d4d3ce84af4bdaaa1a745f189a 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/planprinter/PlanPrinter.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/planprinter/PlanPrinter.java @@ -44,6 +44,8 @@ import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.GroupReference; import io.prestosql.spi.plan.IntersectNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; +import io.prestosql.spi.plan.JoinOnAggregationNode.JoinInternalAggregation; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.OrderingScheme; @@ -468,6 +470,43 @@ public class PlanPrinter return null; } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + List joinExpressions = new ArrayList<>(); + for (JoinNode.EquiJoinClause clause : node.getCriteria()) { + joinExpressions.add(JoinNodeUtils.toExpression(clause).toString()); + } + node.getFilter().map(formatter::apply).ifPresent(joinExpressions::add); + + NodeRepresentation nodeOutput; + nodeOutput = addNode(node, + "Group" + node.getType().getJoinLabel(), + format("[%s]%s", Joiner.on(" AND ").join(joinExpressions), formatHash(node.getLeftHashSymbol(), node.getRightHashSymbol()))); + + node.getDistributionType().ifPresent(distributionType -> nodeOutput.appendDetailsLine("Distribution: %s", distributionType)); + if (!node.getDynamicFilters().isEmpty()) { + nodeOutput.appendDetailsLine("dynamicFilterAssignments = %s", printDynamicFilterAssignments(node.getDynamicFilters())); + } + + Optional sortExpressionContext = node.getFilter().flatMap(filter -> SortExpressionExtractor.extractSortExpression(metadata, node.getRightOutputSymbols(), filter)); + sortExpressionContext.ifPresent(sortContext -> nodeOutput.appendDetailsLine("SortExpression[%s]", formatter.apply(sortContext.getSortExpression()))); + + JoinInternalAggregation aggrOnLeft = node.getAggrOnLeft(); + JoinInternalAggregation aggrOnRight = node.getAggrOnRight(); + nodeOutput.appendDetailsLine("AggrOnAggrLeft = {%s}", getJoinInternalAggregationDetails(aggrOnLeft)); + nodeOutput.appendDetailsLine("AggrOnAggrRight = {%s}", getJoinInternalAggregationDetails(aggrOnRight)); + + JoinInternalAggregation leftAggr = node.getLeftAggr(); + JoinInternalAggregation rightAggr = node.getRightAggr(); + nodeOutput.appendDetailsLine("AggrLeft = {%s}", getJoinInternalAggregationDetails(leftAggr)); + nodeOutput.appendDetailsLine("AggrRight = {%s}", getJoinInternalAggregationDetails(rightAggr)); + + node.getLeft().accept(this, context); + node.getRight().accept(this, context); + return null; + } + @Override public Void visitSpatialJoin(SpatialJoinNode node, Void context) { @@ -1477,6 +1516,36 @@ public class PlanPrinter representation.addNode(nodeOutput); return nodeOutput; } + + private String getJoinInternalAggregationDetails(JoinInternalAggregation node) + { + String type = ""; + if (node.getStep() != AggregationNode.Step.SINGLE) { + type = format("(%s)", node.getStep().toString()); + } + String key = ""; + if (!node.getGroupingKeys().isEmpty()) { + key = node.getGroupingKeys().toString(); + } + String value = ""; + if (node.getAggregationType().equals(AggregationNode.AggregationType.HASH)) { + value = format("HashAggregate%s%s%s", type, key, formatHash(node.getHashSymbol())); + } + StringBuilder builder = new StringBuilder(value); + if (node.getAggregations().size() > 0) { + node.getAggregations().forEach((symbol, aggregation) -> builder.append(format("%s := %s, ", symbol, formatAggregation(aggregation)))); + // to remove the last: ", " + builder.setLength(builder.length() - 2); + } + else { + builder.setLength(builder.length() > 0 ? builder.length() - 1 : 0); + } + String columns = node.getOutputSymbols().stream() + .map(symbol -> symbol + ":" + types.get(symbol).getDisplayName()) + .collect(joining(", ")); + builder.append(" Layout: [").append(columns).append("]"); + return builder.toString(); + } } private static String formatFrame(WindowNode.Frame frame) diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/sanity/DynamicFiltersChecker.java b/presto-main/src/main/java/io/prestosql/sql/planner/sanity/DynamicFiltersChecker.java index 4148a22af127e9f4a94e4e674d56fb30fdafbafe..582720e8990432d0d42b35406bef0416d9b9cff3 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/sanity/DynamicFiltersChecker.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/sanity/DynamicFiltersChecker.java @@ -20,6 +20,7 @@ import io.prestosql.expressions.LogicalRowExpressions; import io.prestosql.metadata.Metadata; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.relation.RowExpression; import io.prestosql.sql.DynamicFilters; @@ -67,7 +68,7 @@ public class DynamicFiltersChecker public Set visitOutput(OutputNode node, List context) { Set unmatched = visitPlan(node, context); - verify(unmatched.isEmpty(), "All consumed dynamic filters could not be matched with a join/semi-join."); + verify(unmatched.isEmpty(), "All consumed dynamic filters could not be matched with a join/semi-join/group-join."); return unmatched; } @@ -90,6 +91,25 @@ public class DynamicFiltersChecker return ImmutableSet.copyOf(unmatched); } + @Override + public Set visitJoinOnAggregation(JoinOnAggregationNode node, List context) + { + Set currentJoinDynamicFilters = node.getDynamicFilters().keySet(); + context.addAll(currentJoinDynamicFilters); + Set consumedProbeSide = node.getLeft().accept(this, context); + verify(difference(currentJoinDynamicFilters, consumedProbeSide).isEmpty(), + "Dynamic filters present in group-join were not fully consumed by it's probe side."); + + context.removeAll(currentJoinDynamicFilters); + Set consumedBuildSide = node.getRight().accept(this, context); + verify(intersection(currentJoinDynamicFilters, consumedBuildSide).isEmpty()); + + Set unmatched = new HashSet<>(consumedBuildSide); + unmatched.addAll(consumedProbeSide); + unmatched.removeAll(currentJoinDynamicFilters); + return ImmutableSet.copyOf(unmatched); + } + @Override public Set visitSemiJoin(SemiJoinNode node, List context) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/sanity/ExternalFunctionPushDownChecker.java b/presto-main/src/main/java/io/prestosql/sql/planner/sanity/ExternalFunctionPushDownChecker.java index f493e1031d42d69999e377086aa852dac4e148d1..f1314da9da97e448240601f7bf91907e8e972e00 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/sanity/ExternalFunctionPushDownChecker.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/sanity/ExternalFunctionPushDownChecker.java @@ -25,6 +25,7 @@ import io.prestosql.spi.connector.CatalogSchemaName; import io.prestosql.spi.plan.AggregationNode; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.ProjectNode; import io.prestosql.spi.plan.ValuesNode; @@ -165,6 +166,16 @@ public class ExternalFunctionPushDownChecker return null; } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Set context) + { + for (PlanNode planNode : node.getSources()) { + planNode.accept(this, context); + } + node.getFilter().ifPresent(rowExpression -> visitRowExpressions(context, rowExpression)); + return null; + } + @Override public Void visitSpatialJoin(SpatialJoinNode node, Set context) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/sanity/ValidateDependenciesChecker.java b/presto-main/src/main/java/io/prestosql/sql/planner/sanity/ValidateDependenciesChecker.java index e7833ab0e65550a87f465cdbab0a342ef44e0336..b9d761e4dc41c9ee0fe9590dba9ede7994f02e25 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/sanity/ValidateDependenciesChecker.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/sanity/ValidateDependenciesChecker.java @@ -26,6 +26,7 @@ import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.IntersectNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; @@ -431,6 +432,43 @@ public final class ValidateDependenciesChecker return null; } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Set boundSymbols) + { + node.getLeft().accept(this, boundSymbols); + node.getRight().accept(this, boundSymbols); + + Set leftInputs = createInputs(node.getLeft(), boundSymbols); + Set rightInputs = createInputs(node.getRight(), boundSymbols); + Set allInputs = ImmutableSet.builder() + .addAll(leftInputs) + .addAll(rightInputs) + .build(); + + for (JoinNode.EquiJoinClause clause : node.getCriteria()) { + checkArgument(leftInputs.contains(clause.getLeft()), "Symbol from join clause (%s) not in left source (%s)", clause.getLeft(), node.getLeft().getOutputSymbols()); + checkArgument(rightInputs.contains(clause.getRight()), "Symbol from join clause (%s) not in right source (%s)", clause.getRight(), node.getRight().getOutputSymbols()); + } + + node.getFilter().ifPresent(predicate -> { + Set predicateSymbols; + if (isExpression(predicate)) { + predicateSymbols = SymbolsExtractor.extractUnique(castToExpression(predicate)); + } + else { + predicateSymbols = SymbolsExtractor.extractUnique(predicate); + } + checkArgument( + allInputs.containsAll(predicateSymbols), + "Symbol from filter (%s) not in sources (%s)", + predicateSymbols, + allInputs); + }); + + checkLeftOutputSymbolsBeforeRight(node.getLeft().getOutputSymbols(), node.getOutputSymbols()); + return null; + } + @Override public Void visitSemiJoin(SemiJoinNode node, Set boundSymbols) { diff --git a/presto-main/src/main/java/io/prestosql/testing/LocalQueryRunner.java b/presto-main/src/main/java/io/prestosql/testing/LocalQueryRunner.java index bb01358906a78590de47ebc1f97aad313291ae58..dc77b75119e384404ccf5a6788331f01600385ad 100644 --- a/presto-main/src/main/java/io/prestosql/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/io/prestosql/testing/LocalQueryRunner.java @@ -119,6 +119,7 @@ import io.prestosql.operator.OutputFactory; import io.prestosql.operator.PagesIndex; import io.prestosql.operator.StageExecutionDescriptor; import io.prestosql.operator.TaskContext; +import io.prestosql.operator.groupjoin.GeneralExecutionHelperFactory; import io.prestosql.operator.index.IndexJoinLookupStats; import io.prestosql.security.GroupProviderManager; import io.prestosql.seedstore.SeedStoreManager; @@ -302,6 +303,7 @@ public class LocalQueryRunner private final CatalogConnectorStore catalogConnectorStore = new CatalogConnectorStore(); private final HeuristicIndexerManager heuristicIndexerManager; private final CubeManager cubeManager; + private final GeneralExecutionHelperFactory executionHelperFactory; private boolean printPlan; private final CachedDataManager cachedDataManager; private final HetuConfig hetuConfig; @@ -534,6 +536,7 @@ public class LocalQueryRunner SpillerStats spillerStats = new SpillerStats(); this.singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(metadata, spillerStats, featuresConfig, nodeSpillConfig, fileSystemClientManager); this.partitioningSpillerFactory = new GenericPartitioningSpillerFactory(this.singleStreamSpillerFactory); + this.executionHelperFactory = new GeneralExecutionHelperFactory(featuresConfig); this.spillerFactory = new GenericSpillerFactory(singleStreamSpillerFactory); this.recoveryUtils = new RecoveryUtils(fileSystemClientManager, new RecoveryConfig(), nodeManager); @@ -862,7 +865,8 @@ public class LocalQueryRunner exchangeManagerRegistry, tableExecuteContextManager, cachedDataManager, - hetuConfig); + hetuConfig, + executionHelperFactory); // plan query StageExecutionDescriptor stageExecutionDescriptor = subplan.getFragment().getStageExecutionDescriptor(); diff --git a/presto-main/src/main/java/io/prestosql/util/GraphvizPrinter.java b/presto-main/src/main/java/io/prestosql/util/GraphvizPrinter.java index b41081d723aa4c5bab2777b1e3657fc701886a8d..56e0c8a1b4c041afef13079b186eaeff17320a73 100644 --- a/presto-main/src/main/java/io/prestosql/util/GraphvizPrinter.java +++ b/presto-main/src/main/java/io/prestosql/util/GraphvizPrinter.java @@ -23,6 +23,7 @@ import io.prestosql.spi.plan.AggregationNode.Aggregation; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.GroupIdNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.LimitNode; import io.prestosql.spi.plan.MarkDistinctNode; import io.prestosql.spi.plan.PlanNode; @@ -106,6 +107,7 @@ public final class GraphvizPrinter INDEX_SOURCE, UNNEST, ANALYZE_FINISH, + GROUPJOIN, } private static final Map NODE_COLORS = immutableEnumMap(ImmutableMap.builder() @@ -130,6 +132,7 @@ public final class GraphvizPrinter .put(NodeType.UNNEST, "crimson") .put(NodeType.SAMPLE, "goldenrod4") .put(NodeType.ANALYZE_FINISH, "plum") + .put(NodeType.GROUPJOIN, "green") .build()); static { @@ -477,6 +480,23 @@ public final class GraphvizPrinter return null; } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + List joinExpressions = new ArrayList<>(); + for (JoinNode.EquiJoinClause clause : node.getCriteria()) { + joinExpressions.add(JoinNodeUtils.toExpression(clause)); + } + + String criteria = Joiner.on(" AND ").join(joinExpressions); + printNode(node, "GROUP" + node.getType().getJoinLabel(), criteria, NODE_COLORS.get(NodeType.GROUPJOIN)); + + node.getLeft().accept(this, context); + node.getRight().accept(this, context); + + return null; + } + @Override public Void visitSemiJoin(SemiJoinNode node, Void context) { diff --git a/presto-main/src/main/java/io/prestosql/utils/DynamicFilterUtils.java b/presto-main/src/main/java/io/prestosql/utils/DynamicFilterUtils.java index d76d10ba28a4f949b54afbcdef7d8c2de91c6fb8..a22877c6d42068a7949fa97babf270a7b9587a62 100644 --- a/presto-main/src/main/java/io/prestosql/utils/DynamicFilterUtils.java +++ b/presto-main/src/main/java/io/prestosql/utils/DynamicFilterUtils.java @@ -18,6 +18,7 @@ import io.prestosql.spi.dynamicfilter.DynamicFilter.DataType; import io.prestosql.spi.dynamicfilter.DynamicFilter.Type; import io.prestosql.spi.plan.FilterNode; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.TableScanNode; import io.prestosql.sql.analyzer.FeaturesConfig.DynamicFilterDataType; @@ -72,6 +73,15 @@ public class DynamicFilterUtils return filterNodes; } + public static List findFilterNodeInStage(JoinOnAggregationNode node) + { + List filterNodes = PlanNodeSearcher + .searchFrom(node.getLeft()) + .where(DynamicFilterUtils::isFilterAboveTableScan) + .findAll(); + return filterNodes; + } + public static List findFilterNodeInStage(SemiJoinNode node) { List filterNodes = PlanNodeSearcher diff --git a/presto-main/src/main/java/io/prestosql/utils/OptimizerUtils.java b/presto-main/src/main/java/io/prestosql/utils/OptimizerUtils.java index 536f90d59dd1e5d2f7b26dc9ae721787b856e19f..75af063ebe55a5bce550e7001f5612f423c031e3 100644 --- a/presto-main/src/main/java/io/prestosql/utils/OptimizerUtils.java +++ b/presto-main/src/main/java/io/prestosql/utils/OptimizerUtils.java @@ -19,6 +19,7 @@ import io.prestosql.Session; import io.prestosql.SystemSessionProperties; import io.prestosql.spi.HetuConstant; import io.prestosql.spi.plan.JoinNode; +import io.prestosql.spi.plan.JoinOnAggregationNode; import io.prestosql.spi.plan.PlanNode; import io.prestosql.spi.plan.TableScanNode; import io.prestosql.sql.analyzer.FeaturesConfig; @@ -216,6 +217,17 @@ public class OptimizerUtils return super.visitJoin(node, context); } + @Override + public Void visitJoinOnAggregation(JoinOnAggregationNode node, Void context) + { + count++; + if (count >= maxLimit) { + // Break once reached the maximum count + return null; + } + return super.visitJoinOnAggregation(node, context); + } + public boolean isMaxCountReached() { return count >= maxLimit; diff --git a/presto-main/src/main/resources/webapp/dist/auditlog.js b/presto-main/src/main/resources/webapp/dist/auditlog.js index 5f2b765b2c1ec2c7971b49d87228d93ac3ec17ae..3d8e0b9ca0683e342e29ac21709e96b8ba1d9a21 100644 --- a/presto-main/src/main/resources/webapp/dist/auditlog.js +++ b/presto-main/src/main/resources/webapp/dist/auditlog.js @@ -23639,7 +23639,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/embedded_plan.js b/presto-main/src/main/resources/webapp/dist/embedded_plan.js index 9985672097f23e42bf69a757b83fcef94667b8fb..d5b18ce992542ddd3583236212ee7ce57a84fd8b 100644 --- a/presto-main/src/main/resources/webapp/dist/embedded_plan.js +++ b/presto-main/src/main/resources/webapp/dist/embedded_plan.js @@ -20542,7 +20542,7 @@ eval("module.exports = function(module) {\n\tif (!module.webpackPolyfill) {\n\t\ /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/hetuqueryeditor.js b/presto-main/src/main/resources/webapp/dist/hetuqueryeditor.js index 795253c330c62d274b24898114f44cda30dcd066..3c554a117908e3af02c5b6aa4576c71d54ee8970 100644 --- a/presto-main/src/main/resources/webapp/dist/hetuqueryeditor.js +++ b/presto-main/src/main/resources/webapp/dist/hetuqueryeditor.js @@ -33028,7 +33028,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/index.js b/presto-main/src/main/resources/webapp/dist/index.js index 74e54743e5f4ef317cfa46fd23609e070cadcd0a..1c2fb386677330c17c74c6ce21c5a98213c99b44 100644 --- a/presto-main/src/main/resources/webapp/dist/index.js +++ b/presto-main/src/main/resources/webapp/dist/index.js @@ -154,7 +154,7 @@ eval("\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/i /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.failed) {\n case \"USER_ERROR\":\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.failed) {\n case \"USER_ERROR\":\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./newUtils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.failed) {\n case \"USER_ERROR\":\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.failed) {\n case \"USER_ERROR\":\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./newUtils.js?"); /***/ }), @@ -23427,7 +23427,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/nodes.js b/presto-main/src/main/resources/webapp/dist/nodes.js index 1ba91b4e3f719c449099a6f2553c79f141f90fab..10547767c6490f992f440577656dc79d56108bd0 100644 --- a/presto-main/src/main/resources/webapp/dist/nodes.js +++ b/presto-main/src/main/resources/webapp/dist/nodes.js @@ -20816,7 +20816,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/overview.js b/presto-main/src/main/resources/webapp/dist/overview.js index 92d5c592e11a86c8af70f28542b401c6bb7f9207..ae885a5be458c5edf35264217a4f13cd563af461 100644 --- a/presto-main/src/main/resources/webapp/dist/overview.js +++ b/presto-main/src/main/resources/webapp/dist/overview.js @@ -26526,7 +26526,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/plan.js b/presto-main/src/main/resources/webapp/dist/plan.js index 06b0df86b0fc079851d40e5c14e09e4b207ccc10..577aed4f07fe3348399f119beeab29595f2d646e 100644 --- a/presto-main/src/main/resources/webapp/dist/plan.js +++ b/presto-main/src/main/resources/webapp/dist/plan.js @@ -20542,7 +20542,7 @@ eval("\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/i /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/query.js b/presto-main/src/main/resources/webapp/dist/query.js index b1eef39bdadba955d3d18fe035697177d7cdc447..d2aec23e32456ba1b75c3af5c28ce7291faba485 100644 --- a/presto-main/src/main/resources/webapp/dist/query.js +++ b/presto-main/src/main/resources/webapp/dist/query.js @@ -20710,7 +20710,7 @@ eval("\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/i /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/querymonitor.js b/presto-main/src/main/resources/webapp/dist/querymonitor.js index 847cff01468b1e87afd23bd283e58174d60087c8..82c6c07dd214172fc4ae43519b9511daac0047d2 100644 --- a/presto-main/src/main/resources/webapp/dist/querymonitor.js +++ b/presto-main/src/main/resources/webapp/dist/querymonitor.js @@ -118,7 +118,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.failed) {\n case \"USER_ERROR\":\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.failed) {\n case \"USER_ERROR\":\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./newUtils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.failed) {\n case \"USER_ERROR\":\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.failed) {\n case \"USER_ERROR\":\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./newUtils.js?"); /***/ }), @@ -23403,7 +23403,7 @@ eval("\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/i /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/stage.js b/presto-main/src/main/resources/webapp/dist/stage.js index c399b1e4ee9b97c8cea339e58c02b52cf91d0701..0737615fccb63481c59b7f0cef2bdabdf3934aaf 100644 --- a/presto-main/src/main/resources/webapp/dist/stage.js +++ b/presto-main/src/main/resources/webapp/dist/stage.js @@ -20542,7 +20542,7 @@ eval("\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/i /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }) diff --git a/presto-main/src/main/resources/webapp/dist/worker.js b/presto-main/src/main/resources/webapp/dist/worker.js index 6388d8af3058435378226fc10bfd44991e11c7b6..39070d44678b65b8cdb3b7f3621d2cc2f081bcdc 100644 --- a/presto-main/src/main/resources/webapp/dist/worker.js +++ b/presto-main/src/main/resources/webapp/dist/worker.js @@ -20506,7 +20506,7 @@ eval("module.exports = function(module) {\n\tif (!module.webpackPolyfill) {\n\t\ /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.GLYPHICON_HIGHLIGHT = exports.GLYPHICON_DEFAULT = undefined;\nexports.getQueryStateColor = getQueryStateColor;\nexports.getStageStateColor = getStageStateColor;\nexports.getHumanReadableState = getHumanReadableState;\nexports.getProgressBarPercentage = getProgressBarPercentage;\nexports.getProgressBarTitle = getProgressBarTitle;\nexports.isQueryEnded = isQueryEnded;\nexports.addToHistory = addToHistory;\nexports.addExponentiallyWeightedToHistory = addExponentiallyWeightedToHistory;\nexports.initializeGraph = initializeGraph;\nexports.initializeSvg = initializeSvg;\nexports.getChildren = getChildren;\nexports.truncateString = truncateString;\nexports.getStageNumber = getStageNumber;\nexports.getTaskIdSuffix = getTaskIdSuffix;\nexports.getTaskNumber = getTaskNumber;\nexports.getFirstParameter = getFirstParameter;\nexports.getHostname = getHostname;\nexports.getPort = getPort;\nexports.getHostAndPort = getHostAndPort;\nexports.computeRate = computeRate;\nexports.precisionRound = precisionRound;\nexports.formatDuration = formatDuration;\nexports.formatRows = formatRows;\nexports.formatCount = formatCount;\nexports.formatDataSizeBytes = formatDataSizeBytes;\nexports.formatDataSize = formatDataSize;\nexports.parseDataSize = parseDataSize;\nexports.parseDuration = parseDuration;\nexports.formatShortTime = formatShortTime;\nexports.formatShortDateTime = formatShortDateTime;\nexports.bubbleSort = bubbleSort;\nexports.removeNodeTypePackage = removeNodeTypePackage;\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\n// Query display\n// =============\n\n/*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nvar GLYPHICON_DEFAULT = exports.GLYPHICON_DEFAULT = { color: '#0e5201' };\nvar GLYPHICON_HIGHLIGHT = exports.GLYPHICON_HIGHLIGHT = { color: '#999999' };\n\nvar STATE_COLOR_MAP = {\n QUEUED: '#1b8f72',\n RUNNING: '#19874e',\n PLANNING: '#674f98',\n FINISHED: '#678975',\n BLOCKED: '#61003b',\n USER_ERROR: '#9a7d66',\n CANCELED: '#858959',\n INSUFFICIENT_RESOURCES: '#7f5b72',\n EXTERNAL_ERROR: '#ca7640',\n UNKNOWN_ERROR: '#943524'\n};\n\nfunction getQueryStateColor(query) {\n switch (query.state) {\n case \"QUEUED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"PLANNING\":\n return STATE_COLOR_MAP.PLANNING;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"STARTING\":\n case \"FINISHING\":\n case \"RUNNING\":\n if (query.queryStats && query.queryStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FAILED\":\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === 'USER_CANCELED') {\n return STATE_COLOR_MAP.CANCELED;\n }\n return STATE_COLOR_MAP.USER_ERROR;\n case \"EXTERNAL\":\n return STATE_COLOR_MAP.EXTERNAL_ERROR;\n case \"INSUFFICIENT_RESOURCES\":\n return STATE_COLOR_MAP.INSUFFICIENT_RESOURCES;\n default:\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n }\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n default:\n return STATE_COLOR_MAP.QUEUED;\n }\n}\n\nfunction getStageStateColor(stage) {\n switch (stage.state) {\n case \"PLANNED\":\n return STATE_COLOR_MAP.QUEUED;\n case \"SUSPENDED\":\n return STATE_COLOR_MAP.BLOCKED;\n case \"SCHEDULING\":\n case \"SCHEDULING_SPLITS\":\n case \"SCHEDULED\":\n return STATE_COLOR_MAP.PLANNING;\n case \"RUNNING\":\n if (stage.stageStats && stage.stageStats.fullyBlocked) {\n return STATE_COLOR_MAP.BLOCKED;\n }\n return STATE_COLOR_MAP.RUNNING;\n case \"FINISHED\":\n return STATE_COLOR_MAP.FINISHED;\n case \"CANCELED\":\n case \"ABORTED\":\n return STATE_COLOR_MAP.CANCELED;\n case \"FAILED\":\n return STATE_COLOR_MAP.UNKNOWN_ERROR;\n default:\n return \"#b5b5b5\";\n }\n}\n\n// This relies on the fact that BasicQueryInfo and QueryInfo have all the fields\n// necessary to compute this string, and that these fields are consistently named.\nfunction getHumanReadableState(query) {\n if (query.state === \"RUNNING\") {\n var title = \"RUNNING\";\n\n if (query.scheduled && query.queryStats.totalDrivers > 0 && query.queryStats.runningDrivers >= 0) {\n if (query.queryStats.fullyBlocked) {\n title = \"BLOCKED\";\n\n if (query.queryStats.blockedReasons && query.queryStats.blockedReasons.length > 0) {\n title += \" (\" + query.queryStats.blockedReasons.join(\", \") + \")\";\n }\n }\n\n if (query.memoryPool === \"reserved\") {\n title += \" (RESERVED)\";\n }\n\n return title;\n }\n }\n\n if (query.state === \"FAILED\") {\n switch (query.errorType) {\n case \"USER_ERROR\":\n if (query.errorCode.name === \"USER_CANCELED\") {\n return \"USER CANCELED\";\n }\n return \"USER ERROR\";\n case \"INTERNAL_ERROR\":\n return \"INTERNAL ERROR\";\n case \"INSUFFICIENT_RESOURCES\":\n return \"INSUFFICIENT RESOURCES\";\n case \"EXTERNAL\":\n return \"EXTERNAL ERROR\";\n }\n }\n\n return query.state;\n}\n\nfunction getProgressBarPercentage(query) {\n var progress = query.queryStats.progressPercentage;\n\n // progress bars should appear 'full' when query progress is not meaningful\n if (!progress || query.state !== \"RUNNING\") {\n return 100;\n }\n\n return Math.round(progress);\n}\n\nfunction getProgressBarTitle(query) {\n if (query.queryStats.progressPercentage && query.state === \"RUNNING\") {\n return getHumanReadableState(query) + \" (\" + getProgressBarPercentage(query) + \"%)\";\n }\n\n return getHumanReadableState(query);\n}\n\nfunction isQueryEnded(query) {\n return [\"FINISHED\", \"FAILED\", \"CANCELED\"].indexOf(query.state) > -1;\n}\n\n// Sparkline-related functions\n// ===========================\n\n// display at most 5 minutes worth of data on the sparklines\nvar MAX_HISTORY = 60 * 5;\n// alpha param of exponentially weighted moving average. picked arbitrarily - lower values means more smoothness\nvar MOVING_AVERAGE_ALPHA = 0.2;\n\nfunction addToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n return valuesArray.concat([value]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\nfunction addExponentiallyWeightedToHistory(value, valuesArray) {\n if (valuesArray.length === 0) {\n return valuesArray.concat([value]);\n }\n\n var movingAverage = value * MOVING_AVERAGE_ALPHA + valuesArray[valuesArray.length - 1] * (1 - MOVING_AVERAGE_ALPHA);\n if (value < 1) {\n movingAverage = 0;\n }\n\n return valuesArray.concat([movingAverage]).slice(Math.max(valuesArray.length - MAX_HISTORY, 0));\n}\n\n// DagreD3 Graph-related functions\n// ===============================\n\nfunction initializeGraph() {\n return new dagreD3.graphlib.Graph({ compound: true }).setGraph({ rankdir: 'BT' }).setDefaultEdgeLabel(function () {\n return {};\n });\n}\n\nfunction initializeSvg(selector) {\n var svg = d3.select(selector);\n svg.append(\"g\");\n\n return svg;\n}\n\nfunction getChildren(nodeInfo) {\n // TODO: Remove this function by migrating StageDetail to use node JSON representation\n var nodeType = removeNodeTypePackage(nodeInfo[\"@type\"]);\n switch (nodeType) {\n case \"OutputNode\":\n case \"ExplainAnalyzeNode\":\n case \"ProjectNode\":\n case \"FilterNode\":\n case \"AggregationNode\":\n case \"SortNode\":\n case \"MarkDistinctNode\":\n case \"WindowNode\":\n case \"RowNumberNode\":\n case \"TopNRowNumberNode\":\n case \"LimitNode\":\n case \"DistinctLimitNode\":\n case \"TopNNode\":\n case \"SampleNode\":\n case \"TableWriterNode\":\n case \"DeleteNode\":\n case 'TableDeleteNode':\n case 'TableFinishNode':\n case 'GroupIdNode':\n case 'CTEScanNode':\n case 'UnnestNode':\n case 'EnforceSingleRowNode':\n return [nodeInfo.source];\n case 'JoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'JoinOnAggregationNode':\n return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source];\n case 'SemiJoinNode':\n return [nodeInfo.source, nodeInfo.filteringSource];\n case 'SpatialJoinNode':\n return [nodeInfo.left, nodeInfo.right];\n case 'IndexJoinNode':\n return [nodeInfo.probeSource, nodeInfo.indexSource];\n case 'UnionNode':\n case 'ExchangeNode':\n return nodeInfo.sources;\n case 'RemoteSourceNode':\n case 'TableScanNode':\n case 'ValuesNode':\n case 'IndexSourceNode':\n break;\n default:\n console.log(\"NOTE: Unhandled PlanNode: \" + nodeType);\n }\n\n return [];\n}\n\n// Utility functions\n// =================\n\nfunction truncateString(inputString, length) {\n if (inputString && inputString.length > length) {\n return inputString.substring(0, length) + \"...\";\n }\n\n return inputString;\n}\n\nfunction getStageNumber(stageId) {\n return Number.parseInt(stageId.slice(stageId.indexOf('.') + 1, stageId.length));\n}\n\nfunction getTaskIdSuffix(taskId) {\n return taskId.slice(taskId.indexOf('.') + 1, taskId.length);\n}\n\nfunction getTaskNumber(taskId) {\n return Number.parseInt(getTaskIdSuffix(getTaskIdSuffix(taskId)));\n}\n\nfunction getFirstParameter(searchString) {\n var searchText = searchString.substring(1);\n\n if (searchText.indexOf('&') !== -1) {\n return searchText.substring(0, searchText.indexOf('&'));\n }\n\n return searchText;\n}\n\nfunction getHostname(url) {\n var hostname = new URL(url).hostname;\n if (hostname.charAt(0) === '[' && hostname.charAt(hostname.length - 1) === ']') {\n hostname = hostname.substr(1, hostname.length - 2);\n }\n return hostname;\n}\n\nfunction getPort(url) {\n return new URL(url).port;\n}\n\nfunction getHostAndPort(urlStr) {\n var url = new URL(urlStr);\n return url.hostname + \":\" + url.port;\n}\n\nfunction computeRate(count, ms) {\n if (ms === 0) {\n return 0;\n }\n return count / ms * 1000.0;\n}\n\nfunction precisionRound(n) {\n if (n < 10) {\n return n.toFixed(2);\n }\n if (n < 100) {\n return n.toFixed(1);\n }\n return Math.round(n).toString();\n}\n\nfunction formatDuration(duration) {\n var unit = \"ms\";\n if (duration > 1000) {\n duration /= 1000;\n unit = \"s\";\n }\n if (unit === \"s\" && duration > 60) {\n duration /= 60;\n unit = \"m\";\n }\n if (unit === \"m\" && duration > 60) {\n duration /= 60;\n unit = \"h\";\n }\n if (unit === \"h\" && duration > 24) {\n duration /= 24;\n unit = \"d\";\n }\n if (unit === \"d\" && duration > 7) {\n duration /= 7;\n unit = \"w\";\n }\n return precisionRound(duration) + unit;\n}\n\nfunction formatRows(count) {\n if (count === 1) {\n return \"1 row\";\n }\n\n return formatCount(count) + \" rows\";\n}\n\nfunction formatCount(count) {\n var unit = \"\";\n if (count > 1000) {\n count /= 1000;\n unit = \"K\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"M\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"B\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"T\";\n }\n if (count > 1000) {\n count /= 1000;\n unit = \"Q\";\n }\n return precisionRound(count) + unit;\n}\n\nfunction formatDataSizeBytes(size) {\n return formatDataSizeMinUnit(size, \"\");\n}\n\nfunction formatDataSize(size) {\n return formatDataSizeMinUnit(size, \"B\");\n}\n\nfunction formatDataSizeMinUnit(size, minUnit) {\n var unit = minUnit;\n if (size === 0) {\n return \"0\" + unit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"K\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"M\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"G\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"T\" + minUnit;\n }\n if (size >= 1024) {\n size /= 1024;\n unit = \"P\" + minUnit;\n }\n return precisionRound(size) + unit;\n}\n\nfunction parseDataSize(value) {\n var DATA_SIZE_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n var match = DATA_SIZE_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"B\":\n return number;\n case \"kB\":\n return number * Math.pow(2, 10);\n case \"MB\":\n return number * Math.pow(2, 20);\n case \"GB\":\n return number * Math.pow(2, 30);\n case \"TB\":\n return number * Math.pow(2, 40);\n case \"PB\":\n return number * Math.pow(2, 50);\n default:\n return null;\n }\n}\n\nfunction parseDuration(value) {\n var DURATION_PATTERN = /^\\s*(\\d+(?:\\.\\d+)?)\\s*([a-zA-Z]+)\\s*$/;\n\n var match = DURATION_PATTERN.exec(value);\n if (match === null) {\n return null;\n }\n var number = parseFloat(match[1]);\n switch (match[2]) {\n case \"ns\":\n return number / 1000000.0;\n case \"us\":\n return number / 1000.0;\n case \"ms\":\n return number;\n case \"s\":\n return number * 1000;\n case \"m\":\n return number * 1000 * 60;\n case \"h\":\n return number * 1000 * 60 * 60;\n case \"d\":\n return number * 1000 * 60 * 60 * 24;\n default:\n return null;\n }\n}\n\nfunction formatShortTime(date) {\n var hours = date.getHours() % 12 || 12;\n var minutes = (date.getMinutes() < 10 ? \"0\" : \"\") + date.getMinutes();\n return hours + \":\" + minutes + (date.getHours() >= 12 ? \"pm\" : \"am\");\n}\n\nfunction formatShortDateTime(date) {\n var year = date.getFullYear();\n var month = \"\" + (date.getMonth() + 1);\n var dayOfMonth = \"\" + date.getDate();\n return year + \"-\" + (month[1] ? month : \"0\" + month[0]) + \"-\" + (dayOfMonth[1] ? dayOfMonth : \"0\" + dayOfMonth[0]) + \" \" + formatShortTime(date);\n}\n\nfunction bubbleSort(arr) {\n var len = arr.length;\n for (var i = 0; i < len - 1; i++) {\n for (var j = 0; j < len - 1 - i; j++) {\n if (arr[j].value < arr[j + 1].value) {\n var temp = arr[j + 1];\n arr[j + 1] = arr[j];\n arr[j] = temp;\n }\n }\n }\n return arr;\n}\n\n// Remove the Java package from each node type to convert the node type to the short name.\n// For example, in the response sent from the server, an output node is represented by\n// \"io.prestosql.sql.planner.plan.OutputNode\". After the invocation of this function,\n// the short name \"OutputNode\" will be returned.\nfunction removeNodeTypePackage(nodeType) {\n var classEndIndex = nodeType.lastIndexOf(\".\");\n return nodeType.substr(classEndIndex + 1);\n}\n\n//# sourceURL=webpack:///./utils.js?"); /***/ }), diff --git a/presto-main/src/main/resources/webapp/src/newUtils.js b/presto-main/src/main/resources/webapp/src/newUtils.js index 9cbc2c099b618508c760a470d458388829c24121..09bbe4ad6abe005350ae18747ed5acfce6243a14 100644 --- a/presto-main/src/main/resources/webapp/src/newUtils.js +++ b/presto-main/src/main/resources/webapp/src/newUtils.js @@ -217,6 +217,8 @@ export function getChildren(nodeInfo: any) return [nodeInfo.source]; case 'JoinNode': return [nodeInfo.left, nodeInfo.right]; + case 'JoinOnAggregationNode': + return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source]; case 'SemiJoinNode': return [nodeInfo.source, nodeInfo.filteringSource]; case 'SpatialJoinNode': diff --git a/presto-main/src/main/resources/webapp/src/utils.js b/presto-main/src/main/resources/webapp/src/utils.js index 9181bdd9ccf5a117a92e6e159bbcca6f051c7e54..fe991a591c1826fd7118562768bc7d6c7a60dff3 100644 --- a/presto-main/src/main/resources/webapp/src/utils.js +++ b/presto-main/src/main/resources/webapp/src/utils.js @@ -245,6 +245,8 @@ export function getChildren(nodeInfo: any) return [nodeInfo.source]; case 'JoinNode': return [nodeInfo.left, nodeInfo.right]; + case 'JoinOnAggregationNode': + return [nodeInfo.leftAggr.source, nodeInfo.rightAggr.source]; case 'SemiJoinNode': return [nodeInfo.source, nodeInfo.filteringSource]; case 'SpatialJoinNode': diff --git a/presto-main/src/test/java/io/prestosql/execution/TaskTestUtils.java b/presto-main/src/test/java/io/prestosql/execution/TaskTestUtils.java index 9a30f28ed2ebdafa60afea1e03ab464e1aa4bb7d..0be425e6c5cb555259e1a4b8af1f82091d4618c2 100644 --- a/presto-main/src/test/java/io/prestosql/execution/TaskTestUtils.java +++ b/presto-main/src/test/java/io/prestosql/execution/TaskTestUtils.java @@ -41,6 +41,7 @@ import io.prestosql.metadata.Split; import io.prestosql.metastore.HetuMetaStoreManager; import io.prestosql.operator.LookupJoinOperators; import io.prestosql.operator.PagesIndex; +import io.prestosql.operator.groupjoin.GeneralExecutionHelperFactory; import io.prestosql.operator.index.IndexJoinLookupStats; import io.prestosql.seedstore.SeedStoreManager; import io.prestosql.spi.connector.CatalogName; @@ -189,7 +190,8 @@ public final class TaskTestUtils cubeManager, new ExchangeManagerRegistry(new ExchangeHandleResolver()), tableExecuteContextManager, - new CachedDataManager(new HetuConfig(), new CacheStorageMonitor(new HetuConfig(), metadata), metadata, null, new SessionPropertyManager()), new HetuConfig()); + new CachedDataManager(new HetuConfig(), new CacheStorageMonitor(new HetuConfig(), metadata), metadata, null, new SessionPropertyManager()), new HetuConfig(), + new GeneralExecutionHelperFactory(featuresConfig)); } public static TaskInfo updateTask(SqlTask sqlTask, List taskSources, OutputBuffers outputBuffers) diff --git a/presto-main/src/test/java/io/prestosql/operator/TestPositionLinks.java b/presto-main/src/test/java/io/prestosql/operator/TestPositionLinks.java index d6b377190b0f0c12b4d4e46879907e4af7377cfc..9b4c30e1b647d6616ea11783a74a33ea1a42dcd2 100644 --- a/presto-main/src/test/java/io/prestosql/operator/TestPositionLinks.java +++ b/presto-main/src/test/java/io/prestosql/operator/TestPositionLinks.java @@ -314,6 +314,7 @@ public class TestPositionLinks ImmutableList.of(), OptionalInt.empty(), Optional.of(0), + Optional.empty(), createTestMetadataManager()); } diff --git a/presto-main/src/test/java/io/prestosql/sql/gen/TestJoinCompiler.java b/presto-main/src/test/java/io/prestosql/sql/gen/TestJoinCompiler.java index a6ad8ff8021b0afda4120fbf7f9d53d96fdf13d7..0ceafe2801e375659864a294a52f88350e56c751 100644 --- a/presto-main/src/test/java/io/prestosql/sql/gen/TestJoinCompiler.java +++ b/presto-main/src/test/java/io/prestosql/sql/gen/TestJoinCompiler.java @@ -191,7 +191,7 @@ public class TestJoinCompiler PagesHashStrategyFactory pagesHashStrategyFactory = tmpJoinCompiler.compilePagesHashStrategyFactory(types, joinChannels, Optional.of(outputChannels)); PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel); // todo add tests for filter function - PagesHashStrategy expectedHashStrategy = new SimplePagesHashStrategy(types, outputChannels, channels, joinChannels, hashChannel, Optional.empty(), metadata); + PagesHashStrategy expectedHashStrategy = new SimplePagesHashStrategy(types, outputChannels, channels, joinChannels, hashChannel, Optional.empty(), Optional.empty(), metadata); // verify channel count assertEquals(hashStrategy.getChannelCount(), outputChannels.size()); diff --git a/presto-spi/src/main/java/io/prestosql/spi/plan/JoinOnAggregationNode.java b/presto-spi/src/main/java/io/prestosql/spi/plan/JoinOnAggregationNode.java new file mode 100644 index 0000000000000000000000000000000000000000..bae2293e81d8d0f4e63774ea9dafc7b1f150ba06 --- /dev/null +++ b/presto-spi/src/main/java/io/prestosql/spi/plan/JoinOnAggregationNode.java @@ -0,0 +1,480 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.spi.plan; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Streams; +import io.prestosql.spi.relation.RowExpression; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.prestosql.spi.plan.AggregationNode.Step.SINGLE; +import static io.prestosql.spi.plan.JoinNode.DistributionType.PARTITIONED; +import static io.prestosql.spi.plan.JoinNode.DistributionType.REPLICATED; +import static io.prestosql.spi.plan.JoinNode.Type.RIGHT; +import static java.util.Objects.requireNonNull; + +public class JoinOnAggregationNode + extends PlanNode +{ + private final JoinNode.Type type; + private final List criteria; + private final Optional filter; + private final Optional leftHashSymbol; + private final Optional rightHashSymbol; + private final Optional distributionType; + private final Optional spillable; + private final Map dynamicFilters; + + private final JoinInternalAggregation leftAggr; + private final JoinInternalAggregation rightAggr; + + private final JoinInternalAggregation aggrOnAggrLeft; + private final JoinInternalAggregation aggrOnAggrRight; + + private final List outputSymbols; + + @JsonCreator + public JoinOnAggregationNode( + @JsonProperty("id") PlanNodeId id, + @JsonProperty("type") JoinNode.Type type, + @JsonProperty("criteria") List criteria, + @JsonProperty("filter") Optional filter, + @JsonProperty("leftHashSymbol") Optional leftHashSymbol, + @JsonProperty("rightHashSymbol") Optional rightHashSymbol, + @JsonProperty("distributionType") Optional distributionType, + @JsonProperty("spillable") Optional spillable, + @JsonProperty("dynamicFilters") Map dynamicFilters, + @JsonProperty("leftAggr") JoinInternalAggregation leftAggr, + @JsonProperty("rightAggr") JoinInternalAggregation rightAggr, + @JsonProperty("aggrOnAggrLeft") JoinInternalAggregation aggrOnAggrLeft, + @JsonProperty("aggrOnAggrRight") JoinInternalAggregation aggrOnAggrRight, + @JsonProperty("outputSymbols") List outputSymbols) + { + super(id); + requireNonNull(type, "type is null"); + requireNonNull(criteria, "criteria is null"); + requireNonNull(outputSymbols, "outputSymbols is null"); + requireNonNull(filter, "filter is null"); + requireNonNull(leftHashSymbol, "leftHashSymbol is null"); + requireNonNull(rightHashSymbol, "rightHashSymbol is null"); + requireNonNull(distributionType, "distributionType is null"); + requireNonNull(spillable, "spillable is null"); + + this.type = type; + this.criteria = ImmutableList.copyOf(criteria); + this.outputSymbols = ImmutableList.copyOf(outputSymbols); + this.filter = filter; + this.leftHashSymbol = leftHashSymbol; + this.rightHashSymbol = rightHashSymbol; + this.distributionType = distributionType; + this.spillable = spillable; + this.dynamicFilters = ImmutableMap.copyOf(requireNonNull(dynamicFilters, "dynamicFilters is null")); + + checkArgument(!(criteria.isEmpty() && leftHashSymbol.isPresent()), "Left hash symbol is only valid in an equijoin"); + checkArgument(!(criteria.isEmpty() && rightHashSymbol.isPresent()), "Right hash symbol is only valid in an equijoin"); + + if (distributionType.isPresent()) { + // The implementation of full outer join only works if the data is hash partitioned. + checkArgument( + !(distributionType.get() == REPLICATED && (type == RIGHT || type == JoinNode.Type.FULL)), + "%s join do not work with %s distribution type", + type, + distributionType.get()); + // It does not make sense to PARTITION when there is nothing to partition on + checkArgument( + !(distributionType.get() == PARTITIONED && criteria.isEmpty() && type != RIGHT && type != JoinNode.Type.FULL), + "Equi criteria are empty, so %s join should not have %s distribution type", + type, + distributionType.get()); + } + + for (Symbol symbol : dynamicFilters.values()) { + checkArgument(rightAggr.getSource().getOutputSymbols().contains(symbol), "Right join input doesn't contain symbol for dynamic filter: %s", symbol); + } + + this.leftAggr = leftAggr; + this.rightAggr = rightAggr; + this.aggrOnAggrLeft = aggrOnAggrLeft; + this.aggrOnAggrRight = aggrOnAggrRight; + } + + @JsonProperty("type") + public JoinNode.Type getType() + { + return type; + } + + public PlanNode getLeft() + { + return leftAggr.getSource(); + } + + public PlanNode getRight() + { + return rightAggr.getSource(); + } + + @JsonProperty("criteria") + public List getCriteria() + { + return criteria; + } + + @JsonProperty("filter") + public Optional getFilter() + { + return filter; + } + + public Set getRightOutputSymbols() + { + return ImmutableSet.copyOf(rightAggr.getSource().getOutputSymbols()); + } + + @JsonProperty("leftHashSymbol") + public Optional getLeftHashSymbol() + { + return leftHashSymbol; + } + + @JsonProperty("rightHashSymbol") + public Optional getRightHashSymbol() + { + return rightHashSymbol; + } + + @Override + public List getSources() + { + return ImmutableList.of(leftAggr.getSource(), rightAggr.getSource()); + } + + @Override + @JsonProperty("outputSymbols") + public List getOutputSymbols() + { + return outputSymbols; + } + + @JsonProperty("distributionType") + public Optional getDistributionType() + { + return distributionType; + } + + @JsonProperty("spillable") + public Optional isSpillable() + { + return spillable; + } + + @JsonProperty + public Map getDynamicFilters() + { + return dynamicFilters; + } + + @JsonProperty("leftAggr") + public JoinInternalAggregation getLeftAggr() + { + return leftAggr; + } + + @JsonProperty("rightAggr") + public JoinInternalAggregation getRightAggr() + { + return rightAggr; + } + + @JsonProperty("aggrOnAggrLeft") + public JoinInternalAggregation getAggrOnLeft() + { + return aggrOnAggrLeft; + } + + @JsonProperty("aggrOnAggrRight") + public JoinInternalAggregation getAggrOnRight() + { + return aggrOnAggrRight; + } + + public List getAllSymbols() + { + return Streams.concat(leftAggr.getAllSymbols().stream(), rightAggr.getAllSymbols().stream(), + aggrOnAggrLeft.getAllSymbols().stream(), aggrOnAggrRight.getAllSymbols().stream()) + .distinct() + .collect(Collectors.toList()); + } + + @Override + public PlanNode replaceChildren(List newChildren) + { + checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); + JoinInternalAggregation leftAggrLocal = (JoinInternalAggregation) this.leftAggr.replaceChildren(newChildren.subList(0, 1)); + JoinInternalAggregation rightAggrLocal = (JoinInternalAggregation) this.rightAggr.replaceChildren(newChildren.subList(1, 2)); + JoinInternalAggregation aggrOnAggrLeftLocal = (JoinInternalAggregation) this.aggrOnAggrLeft.replaceChildren(Collections.singletonList(leftAggrLocal)); + JoinInternalAggregation aggrOnAggrRightLocal = (JoinInternalAggregation) this.aggrOnAggrRight.replaceChildren(Collections.singletonList(rightAggrLocal)); + + return new JoinOnAggregationNode(getId(), + type, + criteria, + filter, + leftHashSymbol, + rightHashSymbol, + distributionType, + spillable, + dynamicFilters, + leftAggrLocal, + rightAggrLocal, + aggrOnAggrLeftLocal, + aggrOnAggrRightLocal, + outputSymbols); + } + + @Override + public R accept(PlanVisitor visitor, C context) + { + return visitor.visitJoinOnAggregation(this, context); + } + + public JoinOnAggregationNode withSpillable(boolean spillable) + { + return new JoinOnAggregationNode(getId(), + type, + criteria, + filter, + leftHashSymbol, + rightHashSymbol, + distributionType, + Optional.of(spillable), + dynamicFilters, + leftAggr, + rightAggr, + aggrOnAggrLeft, + aggrOnAggrRight, + outputSymbols); + } + + public static class JoinInternalAggregation + extends PlanNode + { + private final PlanNode source; + private final Map aggregations; + private final AggregationNode.GroupingSetDescriptor groupingSets; + private final List preGroupedSymbols; + private final AggregationNode.Step step; + private final Optional hashSymbol; + private final Optional groupIdSymbol; + private final List outputs; + private AggregationNode.AggregationType aggregationType; + private Optional finalizeSymbol; + + @JsonCreator + public JoinInternalAggregation( + @JsonProperty("id") PlanNodeId id, + @JsonProperty("source") PlanNode source, + @JsonProperty("aggregations") Map aggregations, + @JsonProperty("groupingSets") AggregationNode.GroupingSetDescriptor groupingSets, + @JsonProperty("preGroupedSymbols") List preGroupedSymbols, + @JsonProperty("step") AggregationNode.Step step, + @JsonProperty("hashSymbol") Optional hashSymbol, + @JsonProperty("groupIdSymbol") Optional groupIdSymbol, + @JsonProperty("aggregationType") AggregationNode.AggregationType aggregationType, + @JsonProperty("finalizeSymbol") Optional finalizeSymbol) + { + super(id); + this.source = source; + this.aggregations = ImmutableMap.copyOf(requireNonNull(aggregations, "aggregations is null")); + + requireNonNull(groupingSets, "groupingSets is null"); + groupIdSymbol.ifPresent(symbol -> checkArgument(groupingSets.getGroupingKeys().contains(symbol), "Grouping columns does not contain groupId column")); + this.groupingSets = groupingSets; + + this.groupIdSymbol = requireNonNull(groupIdSymbol); + + boolean noOrderBy = aggregations.values().stream() + .map(AggregationNode.Aggregation::getOrderingScheme) + .noneMatch(Optional::isPresent); + checkArgument(noOrderBy || step == SINGLE, "ORDER BY does not support distributed aggregation"); + + this.step = step; + this.hashSymbol = hashSymbol; + + requireNonNull(preGroupedSymbols, "preGroupedSymbols is null"); + checkArgument(preGroupedSymbols.isEmpty() || groupingSets.getGroupingKeys().containsAll(preGroupedSymbols), "Pre-grouped symbols must be a subset of the grouping keys"); + this.preGroupedSymbols = ImmutableList.copyOf(preGroupedSymbols); + + ImmutableList.Builder outputs1 = ImmutableList.builder(); + outputs1.addAll(groupingSets.getGroupingKeys()); + hashSymbol.ifPresent(outputs1::add); + outputs1.addAll(aggregations.keySet()); + + AggregationNode.AggregationType tmpAggregationType = aggregationType; + if (tmpAggregationType == AggregationNode.AggregationType.SORT_BASED) { + if (step.equals(AggregationNode.Step.PARTIAL)) { + if (finalizeSymbol.isPresent()) { + List symbolList = new ArrayList<>(Arrays.asList(finalizeSymbol.get())); + outputs1.addAll(symbolList); + } + } + else if (step.equals(AggregationNode.Step.FINAL) && !finalizeSymbol.isPresent()) { + tmpAggregationType = AggregationNode.AggregationType.HASH; + } + } + + this.outputs = outputs1.build(); + this.aggregationType = tmpAggregationType; + this.finalizeSymbol = finalizeSymbol; + } + + @JsonProperty("source") + public PlanNode getSource() + { + return source; + } + + @Override + public List getSources() + { + return ImmutableList.of(source); + } + + public List getOutputSymbols() + { + return outputs; + } + + @Override + public PlanNode replaceChildren(List newChildren) + { + return new JoinInternalAggregation(getId(), + newChildren.get(0), + aggregations, + groupingSets, + preGroupedSymbols, + step, + hashSymbol, + groupIdSymbol, + aggregationType, + finalizeSymbol); + } + + @JsonProperty("step") + public AggregationNode.Step getStep() + { + return step; + } + + @JsonProperty("hashSymbol") + public Optional getHashSymbol() + { + return hashSymbol; + } + + @JsonProperty("groupIdSymbol") + public Optional getGroupIdSymbol() + { + return groupIdSymbol; + } + + @JsonProperty("aggregations") + public Map getAggregations() + { + return aggregations; + } + + @JsonProperty("preGroupedSymbols") + public List getPreGroupedSymbols() + { + return preGroupedSymbols; + } + + @JsonProperty("groupingSets") + public AggregationNode.GroupingSetDescriptor getGroupingSets() + { + return groupingSets; + } + + public List getGroupingKeys() + { + return groupingSets.getGroupingKeys(); + } + + /** + * @return whether this node should produce default output in case of no input pages. + * For example for query: + *

+ * SELECT count(*) FROM nation WHERE nationkey < 0 + *

+ * A default output of "0" is expected to be produced by FINAL aggregation operator. + */ + public boolean hasDefaultOutput() + { + return hasEmptyGroupingSet() && (step.isOutputPartial() || step.equals(SINGLE)); + } + + public boolean hasEmptyGroupingSet() + { + return !groupingSets.getGlobalGroupingSets().isEmpty(); + } + + public boolean hasNonEmptyGroupingSet() + { + return groupingSets.getGroupingSetCount() > groupingSets.getGlobalGroupingSets().size(); + } + + public int getGroupingSetCount() + { + return groupingSets.getGroupingSetCount(); + } + + public Set getGlobalGroupingSets() + { + return groupingSets.getGlobalGroupingSets(); + } + + public boolean hasOrderings() + { + return aggregations.values().stream() + .map(AggregationNode.Aggregation::getOrderingScheme) + .anyMatch(Optional::isPresent); + } + + @JsonProperty("aggregationType") + public AggregationNode.AggregationType getAggregationType() + { + return aggregationType; + } + + @JsonProperty("finalizeSymbol") + public Optional getFinalizeSymbol() + { + return finalizeSymbol; + } + } +} diff --git a/presto-spi/src/main/java/io/prestosql/spi/plan/PlanVisitor.java b/presto-spi/src/main/java/io/prestosql/spi/plan/PlanVisitor.java index 08965aa62100bb30478973b6c7d98a8a7e047e0f..9e3adea2c1cf39ce24024a0b2c63f7623549a32f 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/plan/PlanVisitor.java +++ b/presto-spi/src/main/java/io/prestosql/spi/plan/PlanVisitor.java @@ -47,6 +47,11 @@ public abstract class PlanVisitor return visitPlan(node, context); } + public R visitJoinOnAggregation(JoinOnAggregationNode node, C context) + { + return visitPlan(node, context); + } + public R visitLimit(LimitNode node, C context) { return visitPlan(node, context);