diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateLinearPostprocessOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateLinearPostprocessOperator.java index c805696847..389def201c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateLinearPostprocessOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateLinearPostprocessOperator.java @@ -18,7 +18,9 @@ import java.util.List; import java.util.Objects; -public final class DBSPAggregateLinearPostprocessOperator extends DBSPUnaryOperator { +public final class DBSPAggregateLinearPostprocessOperator + extends DBSPUnaryOperator + implements ILinearAggregate { public final DBSPClosureExpression postProcess; // This operator is incremental-only @@ -27,7 +29,7 @@ public DBSPAggregateLinearPostprocessOperator( DBSPTypeIndexedZSet outputType, DBSPExpression function, DBSPClosureExpression postProcess, OutputPort input) { - super(node, "aggregate_linear_postprocess", function, outputType, false, input, true); + super(node, "aggregate_linear_postprocess", function, outputType, false, input); this.postProcess = postProcess; Utilities.enforce(outputType.elementType.sameType(postProcess.getResultType())); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateLinearPostprocessRetainKeysOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateLinearPostprocessRetainKeysOperator.java index 27dcd5fa92..00e02b91db 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateLinearPostprocessRetainKeysOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateLinearPostprocessRetainKeysOperator.java @@ -17,7 +17,9 @@ import java.util.List; import java.util.Objects; -public final class DBSPAggregateLinearPostprocessRetainKeysOperator extends DBSPBinaryOperator { +public final class DBSPAggregateLinearPostprocessRetainKeysOperator + extends DBSPBinaryOperator + implements ILinearAggregate { public final DBSPClosureExpression postProcess; public final DBSPClosureExpression retainKeysFunction; @@ -30,7 +32,7 @@ public DBSPAggregateLinearPostprocessRetainKeysOperator( DBSPClosureExpression retainKeysFunction, OutputPort left, OutputPort right) { super(node, "aggregate_linear_postprocess_retain_keys", - function, outputType, false, left, right, true); + function, outputType, false, left, right); this.postProcess = postProcess; this.retainKeysFunction = retainKeysFunction; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateOperator.java index 17c6cb3806..b1513d1eb1 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateOperator.java @@ -40,14 +40,13 @@ import javax.annotation.Nullable; import java.util.List; -public final class DBSPAggregateOperator extends DBSPAggregateOperatorBase { +public final class DBSPAggregateOperator extends DBSPAggregateOperatorBase implements INonLinearAggregate { public DBSPAggregateOperator( CalciteRelNode node, DBSPTypeIndexedZSet outputType, @Nullable DBSPAggregator function, @Nullable DBSPAggregateList aggregate, OutputPort input) { - super(node, "aggregate", outputType, - function, aggregate, false, input, true); + super(node, "aggregate", outputType, function, aggregate, false, input); Utilities.enforce(input.getOutputIndexedZSetType().keyType.sameType(outputType.keyType)); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateOperatorBase.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateOperatorBase.java index 417a771e63..2369d7a9c2 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateOperatorBase.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateOperatorBase.java @@ -14,7 +14,8 @@ import java.util.Objects; /*** Base class for operators that perform some form of aggregation. */ -public abstract class DBSPAggregateOperatorBase extends DBSPUnaryOperator { +public abstract class DBSPAggregateOperatorBase + extends DBSPUnaryOperator { // Initially 'aggregateList' may be not null, and 'function' may be null. // Later always aggregateList is null, and function is not null. @Nullable @@ -25,9 +26,8 @@ protected DBSPAggregateOperatorBase(CalciteRelNode node, String operation, @Nullable DBSPAggregator function, @Nullable DBSPAggregateList aggregateList, boolean multiset, - OutputPort source, - boolean containsIntegrate) { - super(node, operation, function, outputType, multiset, source, containsIntegrate); + OutputPort source) { + super(node, operation, function, outputType, multiset, source); this.aggregateList = aggregateList; // There are really two different representations of an aggregate operator, // which reuse the same classes: a high-level one, which contains an Aggregate, diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateZeroOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateZeroOperator.java index f7c32dc34b..d50559900e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateZeroOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAggregateZeroOperator.java @@ -41,7 +41,7 @@ * | */ @NonCoreIR -public class DBSPAggregateZeroOperator extends DBSPUnaryOperator { +public class DBSPAggregateZeroOperator extends DBSPUnaryOperator implements ILinear { /** Create an AggregateZero operator. * * @param node Calcite node. diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAntiJoinOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAntiJoinOperator.java index f518c881eb..4a1a9e0b6c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAntiJoinOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPAntiJoinOperator.java @@ -16,9 +16,9 @@ /** Anti join operator: produces elements from the left stream that have no corresponding keys * in the right stream. */ -public final class DBSPAntiJoinOperator extends DBSPBinaryOperator { +public final class DBSPAntiJoinOperator extends DBSPBinaryOperator implements IJoin { public DBSPAntiJoinOperator(CalciteRelNode node, OutputPort left, OutputPort right) { - super(node, "antijoin", null, left.outputType(), left.isMultiset(), left, right, true); + super(node, "antijoin", null, left.outputType(), left.isMultiset(), left, right); // Inputs must be indexed left.getOutputIndexedZSetType(); right.getOutputIndexedZSetType(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPApply2Operator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPApply2Operator.java index 8e50a74782..0c3f5cf827 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPApply2Operator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPApply2Operator.java @@ -19,10 +19,10 @@ * which applies an arbitrary function to its 2 inputs. * The inputs and outputs cannot be Z-sets or indexed Z-sets. * The comments from {@link DBSPApplyOperator} apply to this operator as well. */ -public final class DBSPApply2Operator extends DBSPBinaryOperator { +public final class DBSPApply2Operator extends DBSPBinaryOperator implements ILinear { public DBSPApply2Operator(CalciteRelNode node, DBSPClosureExpression function, OutputPort left, OutputPort right) { - super(node, "apply2", function, function.getResultType(), false, left, right, false); + super(node, "apply2", function, function.getResultType(), false, left, right); Utilities.enforce(function.parameters.length == 2, () -> "Expected 2 parameters for function " + function); DBSPType param0Type = function.parameters[0].getType().deref(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPApplyOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPApplyOperator.java index 49619d2736..b8bac2c8e4 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPApplyOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPApplyOperator.java @@ -27,7 +27,7 @@ * from a {@link DBSPWaterlineOperator}, which replicates its output to all workers. * So it's never OK to have an apply operator process inputs from standard operators. * In the type system such inputs would show up as ZSets or IndexedZSets. */ -public final class DBSPApplyOperator extends DBSPUnaryOperator { +public final class DBSPApplyOperator extends DBSPUnaryOperator implements ILinear { public static void noZsets(DBSPType type) { Utilities.enforce(!type.is(DBSPTypeZSet.class)); Utilities.enforce(!type.is(DBSPTypeIndexedZSet.class)); @@ -35,7 +35,7 @@ public static void noZsets(DBSPType type) { public DBSPApplyOperator(CalciteRelNode node, DBSPClosureExpression function, DBSPType outputType, OutputPort input, @Nullable String comment) { - super(node, "apply", function, outputType, false, input, comment, false); + super(node, "apply", function, outputType, false, input, comment); Utilities.enforce(function.parameters.length == 1, () -> "Expected 1 parameter for function " + function); DBSPType paramType = function.parameters[0].getType().deref(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPBinaryOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPBinaryOperator.java index c2bee7b6d3..6f23a2a940 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPBinaryOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPBinaryOperator.java @@ -11,9 +11,8 @@ public abstract class DBSPBinaryOperator extends DBSPSimpleOperator { protected DBSPBinaryOperator(CalciteRelNode node, String operation, @Nullable DBSPExpression function, DBSPType outputType, - boolean isMultiset, OutputPort left, OutputPort right, - boolean containsIntegrator) { - super(node, operation, function, outputType, isMultiset, containsIntegrator); + boolean isMultiset, OutputPort left, OutputPort right) { + super(node, operation, function, outputType, isMultiset); this.addInput(left); this.addInput(right); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPChainAggregateOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPChainAggregateOperator.java index fc9b90f91b..28c0352f38 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPChainAggregateOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPChainAggregateOperator.java @@ -20,12 +20,12 @@ /** Corresponds to the DBSP chain_aggregate operator. * Used to implement min and max for append-only collections with O(1) space and time. */ -public class DBSPChainAggregateOperator extends DBSPUnaryOperator { +public class DBSPChainAggregateOperator extends DBSPUnaryOperator implements ILinearAggregate { public final DBSPClosureExpression init; public DBSPChainAggregateOperator(CalciteRelNode node, DBSPClosureExpression init, DBSPClosureExpression function, DBSPType outputType, OutputPort source) { - super(node, "chain_aggregate", function, outputType, false, source, true); + super(node, "chain_aggregate", function, outputType, false, source); this.init = init; Utilities.enforce(init.parameters.length == 2); Utilities.enforce(function.parameters.length == 3); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPChainOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPChainOperator.java index 46985bbf9d..be41b6d6b1 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPChainOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPChainOperator.java @@ -38,7 +38,7 @@ /** A Chain operator performs a linear chain of Map/Filter/MapIndex operations. * In the end it is lowered to a {@link DBSPFlatMapOperator} or * {@link DBSPFlatMapIndexOperator} operator, depending on the last operation. */ -public class DBSPChainOperator extends DBSPUnaryOperator { +public class DBSPChainOperator extends DBSPUnaryOperator implements ILinear { public final ComputationChain chain; public DBSPChainOperator(CalciteRelNode node, ComputationChain chain, boolean isMultiset, OutputPort source) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPConstantOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPConstantOperator.java index d9f53fc3e7..037d2aa36b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPConstantOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPConstantOperator.java @@ -41,7 +41,7 @@ public final class DBSPConstantOperator extends DBSPSimpleOperator { public DBSPConstantOperator(CalciteRelNode node, DBSPExpression value, boolean isMultiset) { // Notice that we use the 'this.function' field to represent // the constant value. Constants are not ClosureExpressions. - super(node, "constant", value, value.getType(), isMultiset, false); + super(node, "constant", value, value.getType(), isMultiset); } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPControlledKeyFilterOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPControlledKeyFilterOperator.java index 2a714e5c22..efb65e5822 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPControlledKeyFilterOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPControlledKeyFilterOperator.java @@ -37,7 +37,7 @@ * the input element makes it to the output. The {@code error} function takes * a scalar, and an element and returns an error message with the ERROR_SCHEMA structure. * It is invoked only when the function returns 'false'. */ -public final class DBSPControlledKeyFilterOperator extends DBSPOperatorWithError { +public final class DBSPControlledKeyFilterOperator extends DBSPOperatorWithError implements ILinear { public DBSPControlledKeyFilterOperator( CalciteRelNode node, DBSPClosureExpression function, DBSPClosureExpression error, OutputPort data, OutputPort control) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDeindexOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDeindexOperator.java index 8f19c1b571..a91877fc6f 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDeindexOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDeindexOperator.java @@ -20,7 +20,7 @@ /** The inverse of the map_index operator. This operator simply drops the index * from an indexed z-set and keeps all the values. It can be implemented by * a DBSP map operator, but it is worthwhile to have a separate operation. */ -public final class DBSPDeindexOperator extends DBSPUnaryOperator { +public final class DBSPDeindexOperator extends DBSPUnaryOperator implements ILinear { static DBSPType outputType(DBSPTypeIndexedZSet ix) { return TypeCompiler.makeZSet(ix.elementType); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDelayOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDelayOperator.java index fbfeedd044..91f5622400 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDelayOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDelayOperator.java @@ -16,7 +16,7 @@ /** The z^-1 operator from DBSP. * If the function is specified, it is the initial value produced by the delay. */ -public final class DBSPDelayOperator extends DBSPUnaryOperator { +public final class DBSPDelayOperator extends DBSPUnaryOperator implements IStateful { public DBSPDelayOperator(CalciteRelNode node, @Nullable DBSPExpression initial, OutputPort source) { super(node, initial == null ? "transaction_delay" : "transaction_delay_with_initial_value", initial, source.outputType(), source.isMultiset(), source); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDelayedIntegralOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDelayedIntegralOperator.java index 77b38e6527..ebbe8c0a4b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDelayedIntegralOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDelayedIntegralOperator.java @@ -18,7 +18,7 @@ * This shows up often, and it can be implemented more efficiently * than using the pair. */ @NonCoreIR -public final class DBSPDelayedIntegralOperator extends DBSPUnaryOperator { +public final class DBSPDelayedIntegralOperator extends DBSPUnaryOperator implements IContainsIntegrator { public DBSPDelayedIntegralOperator(CalciteRelNode node, OutputPort source) { super(node, "accumulate_delay_trace", null, source.outputType(), source.isMultiset(), source); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDeltaOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDeltaOperator.java index c308461661..b227f3f4d0 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDeltaOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDeltaOperator.java @@ -14,7 +14,7 @@ import java.util.List; /** Represents a delta operator (called delta0 in DBSP) */ -public class DBSPDeltaOperator extends DBSPUnaryOperator { +public class DBSPDeltaOperator extends DBSPUnaryOperator implements ILinear { public DBSPDeltaOperator(CalciteRelNode node, OutputPort source) { super(node, "delta0", null, source.outputType(), source.isMultiset(), source); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDifferentiateOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDifferentiateOperator.java index 32629f67e7..d4f3c90593 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDifferentiateOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDifferentiateOperator.java @@ -36,7 +36,7 @@ import javax.annotation.Nullable; import java.util.List; -public final class DBSPDifferentiateOperator extends DBSPUnaryOperator { +public final class DBSPDifferentiateOperator extends DBSPUnaryOperator implements IStateful { public DBSPDifferentiateOperator(CalciteRelNode node, OutputPort source) { super(node, "differentiate", null, source.outputType(), source.isMultiset(), source); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDistinctIncrementalOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDistinctIncrementalOperator.java index 4b91800541..9fed67ba48 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDistinctIncrementalOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDistinctIncrementalOperator.java @@ -16,9 +16,9 @@ import java.util.List; @NonCoreIR -public final class DBSPDistinctIncrementalOperator extends DBSPBinaryOperator { +public final class DBSPDistinctIncrementalOperator extends DBSPBinaryOperator implements IContainsIntegrator { public DBSPDistinctIncrementalOperator(CalciteRelNode node, OutputPort integral, OutputPort delta) { - super(node, "distinct_incremental", null, delta.outputType(), false, integral, delta, false); + super(node, "distinct_incremental", null, delta.outputType(), false, integral, delta); } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDistinctOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDistinctOperator.java index 25b7c5c0fa..6935e9b627 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDistinctOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPDistinctOperator.java @@ -36,9 +36,9 @@ import javax.annotation.Nullable; import java.util.List; -public final class DBSPDistinctOperator extends DBSPUnaryOperator { +public final class DBSPDistinctOperator extends DBSPUnaryOperator implements IContainsIntegrator { public DBSPDistinctOperator(CalciteRelNode node, OutputPort input) { - super(node, "distinct", null, input.outputType(), false, input, true); + super(node, "distinct", null, input.outputType(), false, input); } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFilterOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFilterOperator.java index 4609625304..94b4986f9a 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFilterOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFilterOperator.java @@ -38,7 +38,7 @@ import java.util.List; import java.util.Objects; -public final class DBSPFilterOperator extends DBSPUnaryOperator { +public final class DBSPFilterOperator extends DBSPUnaryOperator implements ILinear { public DBSPFilterOperator(CalciteRelNode node, DBSPExpression condition, OutputPort input) { super(node, "filter", condition, input.outputType(), input.isMultiset(), input); this.checkResultType(condition, new DBSPTypeBool(CalciteEmptyRel.INSTANCE, false)); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFlatMapIndexOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFlatMapIndexOperator.java index 95c0014b34..11ddbd69ea 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFlatMapIndexOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFlatMapIndexOperator.java @@ -15,7 +15,7 @@ import java.util.List; import java.util.Objects; -public final class DBSPFlatMapIndexOperator extends DBSPUnaryOperator { +public final class DBSPFlatMapIndexOperator extends DBSPUnaryOperator implements ILinear { public DBSPFlatMapIndexOperator(CalciteRelNode node, DBSPExpression expression, DBSPTypeIndexedZSet outputType, boolean isMultiset, OutputPort input) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFlatMapOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFlatMapOperator.java index 14e8bffbbc..d037060f53 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFlatMapOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPFlatMapOperator.java @@ -38,7 +38,7 @@ import java.util.List; import java.util.Objects; -public final class DBSPFlatMapOperator extends DBSPUnaryOperator { +public final class DBSPFlatMapOperator extends DBSPUnaryOperator implements ILinear { public DBSPFlatMapOperator(CalciteRelNode node, DBSPExpression expression, DBSPTypeZSet outputType, boolean isMultiset, OutputPort input) { super(node, "flat_map", expression, outputType, isMultiset, input); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPHopOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPHopOperator.java index 517c99476d..3c08c4fa52 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPHopOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPHopOperator.java @@ -21,7 +21,7 @@ * operators: map (computing the hop start window) followed by flat_map * (which generates all the windows). It does not correspond to any DBSP * Rust operator. */ -public final class DBSPHopOperator extends DBSPUnaryOperator { +public final class DBSPHopOperator extends DBSPUnaryOperator implements ILinear { public final int timestampIndex; public final DBSPExpression interval; public final DBSPExpression start; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIndexedTopKOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIndexedTopKOperator.java index 1342b22e12..34f0ca5344 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIndexedTopKOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIndexedTopKOperator.java @@ -27,7 +27,7 @@ /** Apply a topK operation to each of the groups in an indexed collection. * This always sorts the elements of each group. * To sort the entire collection just group by (). */ -public final class DBSPIndexedTopKOperator extends DBSPUnaryOperator { +public final class DBSPIndexedTopKOperator extends DBSPUnaryOperator implements IContainsIntegrator { /** These values correspond to the SQL keywords * ROW, RANK, and DENSE RANK. See e.g.: * https://learn.microsoft.com/en-us/sql/t-sql/functions/ranking-functions-transact-sql @@ -75,7 +75,7 @@ public DBSPIndexedTopKOperator(CalciteRelNode node, TopKNumbering numbering, DBSPEqualityComparatorExpression equalityComparator, DBSPClosureExpression outputProducer, OutputPort source) { super(node, "topK", comparator, - outputType(source.getOutputIndexedZSetType(), outputProducer), source.isMultiset(), source, true); + outputType(source.getOutputIndexedZSetType(), outputProducer), source.isMultiset(), source); Utilities.enforce(comparator.is(DBSPComparatorExpression.class) || comparator.is(DBSPPathExpression.class)); this.limit = limit; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInputMapWithWaterlineOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInputMapWithWaterlineOperator.java index 8f7482449c..eab838b2db 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInputMapWithWaterlineOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInputMapWithWaterlineOperator.java @@ -1,8 +1,6 @@ package org.dbsp.sqlCompiler.circuit.operator; import com.fasterxml.jackson.databind.JsonNode; -import org.dbsp.sqlCompiler.circuit.IInputMapOperator; -import org.dbsp.sqlCompiler.circuit.IMultiOutput; import org.dbsp.sqlCompiler.circuit.OutputPort; import org.dbsp.sqlCompiler.compiler.TableMetadata; import org.dbsp.sqlCompiler.compiler.backend.JsonDecoder; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateOperator.java index f0bf344195..f5cc63f869 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateOperator.java @@ -36,7 +36,7 @@ import javax.annotation.Nullable; import java.util.List; -public final class DBSPIntegrateOperator extends DBSPUnaryOperator { +public final class DBSPIntegrateOperator extends DBSPUnaryOperator implements IContainsIntegrator { public DBSPIntegrateOperator(CalciteRelNode node, OutputPort source) { super(node, "integrate", null, source.outputType(), source.isMultiset(), source); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java index 754aced250..95df2418c4 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainKeysOperator.java @@ -25,13 +25,13 @@ import java.util.Objects; public final class DBSPIntegrateTraceRetainKeysOperator - extends DBSPBinaryOperator implements GCOperator + extends DBSPBinaryOperator implements IGCOperator { public DBSPIntegrateTraceRetainKeysOperator( CalciteRelNode node, DBSPExpression expression, OutputPort data, OutputPort control) { super(node, "accumulate_integrate_trace_retain_keys", expression, - data.outputType(), data.isMultiset(), data, control, false); + data.outputType(), data.isMultiset(), data, control); } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainNValuesOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainNValuesOperator.java index fb232a2ca2..b80e5f2a83 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainNValuesOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainNValuesOperator.java @@ -31,7 +31,7 @@ * - bottom_n * Used to GC the right input of ASOF JOIN operators, Min, Max, ArgMin, ArgMax, MinSome, TopK. */ public final class DBSPIntegrateTraceRetainNValuesOperator - extends DBSPBinaryOperator implements GCOperator { + extends DBSPBinaryOperator implements IGCOperator { public enum WhichN { LastN("last_n"), TopN("top_n"), @@ -51,7 +51,7 @@ public DBSPIntegrateTraceRetainNValuesOperator( CalciteRelNode node, DBSPExpression function, OutputPort data, OutputPort control, int n, WhichN which) { super(node, "accumulate_integrate_trace_retain_values_" + which.which, - function, data.outputType(), data.isMultiset(), data, control, false); + function, data.outputType(), data.isMultiset(), data, control); Utilities.enforce(n > 0); this.n = n; this.which = which; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java index 9e2c74d5e9..b5a11a48ad 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPIntegrateTraceRetainValuesOperator.java @@ -25,7 +25,7 @@ /** Currently always inserted after the input of a join operator */ public final class DBSPIntegrateTraceRetainValuesOperator - extends DBSPBinaryOperator implements GCOperator { + extends DBSPBinaryOperator implements IGCOperator { public final boolean accumulate; @@ -33,7 +33,7 @@ public DBSPIntegrateTraceRetainValuesOperator( CalciteRelNode node, DBSPExpression function, OutputPort data, OutputPort control, boolean accumulate) { super(node, accumulate ? "accumulate_integrate_trace_retain_values" : "integrate_trace_retain_values", - function, data.outputType(), data.isMultiset(), data, control, false); + function, data.outputType(), data.isMultiset(), data, control); this.accumulate = accumulate; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInternOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInternOperator.java index c84c99633e..0962e254b7 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInternOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPInternOperator.java @@ -16,7 +16,7 @@ /** An operator whose input contains all interned strings. * Behaves like a sink which is always live. * Does not correspond to a DBSP operator. */ -public class DBSPInternOperator extends DBSPUnaryOperator { +public class DBSPInternOperator extends DBSPUnaryOperator implements ILinear { public DBSPInternOperator(OutputPort source) { super(CalciteEmptyRel.INSTANCE, "interned_strings", null, source.outputType(), false, source); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPJoinBaseOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPJoinBaseOperator.java index 5c2e8ffd02..364e90f91c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPJoinBaseOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPJoinBaseOperator.java @@ -11,7 +11,7 @@ /** Base class for the many kinds of joins we have, some incremental, * some non-incremental. */ -public abstract class DBSPJoinBaseOperator extends DBSPBinaryOperator { +public abstract class DBSPJoinBaseOperator extends DBSPBinaryOperator implements IJoin { /** If true the join is dynamically balanced; else it is always a hash-join */ public final boolean balanced; @@ -19,7 +19,7 @@ protected DBSPJoinBaseOperator( CalciteRelNode node, String operation, DBSPExpression function, DBSPType outputType, boolean isMultiset, OutputPort left, OutputPort right, boolean balanced) { - super(node, operation, function, outputType, isMultiset, left, right, true); + super(node, operation, function, outputType, isMultiset, left, right); this.checkParameterCount(function, 3); this.balanced = balanced; DBSPClosureExpression closure = this.getClosureFunction(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPLagOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPLagOperator.java index d698bd368a..0d8ea1f655 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPLagOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPLagOperator.java @@ -21,7 +21,7 @@ /** Implements the LAG/LEAD operators for an SQL OVER Window. The LEAD/LAG operator * is an interesting WINDOW operator, since it computes the entire content of the window. * The output of other window aggregates need to be joined with the data, but this one doesn't. */ -public final class DBSPLagOperator extends DBSPUnaryOperator { +public final class DBSPLagOperator extends DBSPUnaryOperator implements IContainsIntegrator { // Usually a DBSPComparatorExpression, replaced with a PathExpression later. public final DBSPExpression comparator; public final DBSPExpression projection; @@ -43,7 +43,7 @@ public DBSPLagOperator(CalciteRelNode node, int offset, DBSPExpression projection, DBSPExpression function, DBSPExpression comparator, DBSPTypeIndexedZSet outputType, OutputPort source) { - super(node, "lag_custom_order", function, outputType, source.isMultiset(), source, true); + super(node, "lag_custom_order", function, outputType, source.isMultiset(), source); Utilities.enforce(comparator.is(DBSPComparatorExpression.class) || comparator.is(DBSPPathExpression.class)); if (comparator.is(DBSPComparatorExpression.class)) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPMapIndexOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPMapIndexOperator.java index abdd3704ae..0181efce5e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPMapIndexOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPMapIndexOperator.java @@ -42,7 +42,7 @@ /** Generate an IndexedZSet by applying a function to every element of an input dataset. * Output is always an IndexedZSet. Input can be a ZSet or an IndexedZSet */ -public final class DBSPMapIndexOperator extends DBSPUnaryOperator { +public final class DBSPMapIndexOperator extends DBSPUnaryOperator implements ILinear { /** Create an MapIndexOperator * @param node Corresponding Calcite node. * @param indexFunction Function that indexes. diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPMapOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPMapOperator.java index 17519cc34e..0b22f5c105 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPMapOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPMapOperator.java @@ -39,7 +39,7 @@ import java.util.List; import java.util.Objects; -public final class DBSPMapOperator extends DBSPUnaryOperator { +public final class DBSPMapOperator extends DBSPUnaryOperator implements ILinear { public DBSPMapOperator(CalciteRelNode node, DBSPExpression function, DBSPTypeZSet outputType, boolean isMultiset, OutputPort input) { super(node, "map", function, outputType, isMultiset, input); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNegateOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNegateOperator.java index b793fd0254..46a26a99d1 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNegateOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNegateOperator.java @@ -36,7 +36,7 @@ import javax.annotation.Nullable; import java.util.List; -public final class DBSPNegateOperator extends DBSPUnaryOperator { +public final class DBSPNegateOperator extends DBSPUnaryOperator implements ILinear { public DBSPNegateOperator(CalciteRelNode node, OutputPort input) { super(node, "neg", null, input.outputType(), input.isMultiset(), input); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNoopOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNoopOperator.java index 9bb7a5b4a1..cbe5c33ad9 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNoopOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNoopOperator.java @@ -44,7 +44,7 @@ /** Same as a map with identity function, but, unlike a {@link DBSPMapOperator}, * the output can be an IndexedZSet. */ -public final class DBSPNoopOperator extends DBSPUnaryOperator { +public final class DBSPNoopOperator extends DBSPUnaryOperator implements ILinear { static DBSPClosureExpression getClosure(DBSPType sourceType) { if (sourceType.is(DBSPTypeZSet.class)) { DBSPVariablePath var = sourceType.to(DBSPTypeZSet.class).elementType.ref().var(); @@ -64,7 +64,7 @@ static DBSPClosureExpression getClosure(DBSPType sourceType) { public DBSPNoopOperator(CalciteRelNode node, OutputPort source, @Nullable String comment) { super(node, "noop", getClosure(source.outputType()), - source.outputType(), source.isMultiset(), source, comment, false); + source.outputType(), source.isMultiset(), source, comment); } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNowOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNowOperator.java index 3816532bfd..68a060d101 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNowOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPNowOperator.java @@ -24,7 +24,7 @@ * There is no equivalent DBSP operator, this is only used during compilation to * represent an input which is connected to a stream containing clock values. * (The compiler option options.ioOptions.nowStream controls how this is implemented). */ -public final class DBSPNowOperator extends DBSPSimpleOperator { +public final class DBSPNowOperator extends DBSPSimpleOperator implements ILinear { static DBSPExpression createFunction(CalciteObject node) { return new DBSPZSetExpression( new DBSPTupleExpression(new DBSPApplyExpression( @@ -33,8 +33,7 @@ static DBSPExpression createFunction(CalciteObject node) { public DBSPNowOperator(CalciteRelNode node) { super(node, "now", createFunction(node), - new DBSPTypeZSet(new DBSPTypeTuple(DBSPTypeTimestamp.create(node, false))), - false, false); + new DBSPTypeZSet(new DBSPTypeTuple(DBSPTypeTimestamp.create(node, false))), false); } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPOperatorWithError.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPOperatorWithError.java index be08de690c..2c9f02b187 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPOperatorWithError.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPOperatorWithError.java @@ -1,6 +1,5 @@ package org.dbsp.sqlCompiler.circuit.operator; -import org.dbsp.sqlCompiler.circuit.IMultiOutput; import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteRelNode; import org.dbsp.sqlCompiler.compiler.visitors.inner.EquivalenceContext; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPartitionedRollingAggregateOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPartitionedRollingAggregateOperator.java index fc94e7c891..b9ff0f44d6 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPartitionedRollingAggregateOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPartitionedRollingAggregateOperator.java @@ -27,7 +27,8 @@ /** This operator only operates correctly on deltas. To operate on collections it * must differentiate its input, and integrate its output. */ -public final class DBSPPartitionedRollingAggregateOperator extends DBSPAggregateOperatorBase { +public final class DBSPPartitionedRollingAggregateOperator extends DBSPAggregateOperatorBase + implements INonLinearAggregate { public final DBSPClosureExpression partitioningFunction; public final DBSPWindowBoundExpression lower; public final DBSPWindowBoundExpression upper; @@ -46,7 +47,7 @@ public DBSPPartitionedRollingAggregateOperator( // the current IR, so this type is a lie. DBSPTypeIndexedZSet outputType, OutputPort input) { - super(node, "partitioned_rolling_aggregate", outputType, function, aggregate, true, input, true); + super(node, "partitioned_rolling_aggregate", outputType, function, aggregate, true, input); this.lower = lower; this.upper = upper; this.partitioningFunction = partitioningFunction; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPartitionedRollingAggregateWithWaterlineOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPartitionedRollingAggregateWithWaterlineOperator.java index 5177c8b32f..81ac9fe84b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPartitionedRollingAggregateWithWaterlineOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPartitionedRollingAggregateWithWaterlineOperator.java @@ -48,7 +48,8 @@ import java.util.List; public final class DBSPPartitionedRollingAggregateWithWaterlineOperator - extends DBSPBinaryOperator { + extends DBSPBinaryOperator + implements INonLinearAggregate { @Nullable public final DBSPAggregateList aggregateList; public final DBSPClosureExpression partitioningFunction; @@ -71,7 +72,7 @@ public DBSPPartitionedRollingAggregateWithWaterlineOperator( DBSPTypeIndexedZSet outputType, OutputPort dataInput, OutputPort waterlineInput) { super(node, "partitioned_rolling_aggregate_with_waterline", - function, outputType, true, dataInput, waterlineInput, true); + function, outputType, true, dataInput, waterlineInput); this.aggregateList = aggregateList; this.lower = lower; this.upper = upper; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPrimitiveAggregateOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPrimitiveAggregateOperator.java index 1e0657b568..4a55a65a23 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPrimitiveAggregateOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPPrimitiveAggregateOperator.java @@ -17,11 +17,11 @@ /** This is a primitive operator that corresponds to the Rust AggregateIncremental node. */ @NonCoreIR -public final class DBSPPrimitiveAggregateOperator extends DBSPBinaryOperator { +public final class DBSPPrimitiveAggregateOperator extends DBSPBinaryOperator implements INonLinearAggregate { public DBSPPrimitiveAggregateOperator( CalciteRelNode node, @Nullable DBSPExpression function, DBSPType outputType, OutputPort delta, OutputPort integral) { - super(node, "AggregateIncremental", function, outputType, false, delta, integral, true); + super(node, "AggregateIncremental", function, outputType, false, delta, integral); Utilities.enforce(delta.getOutputIndexedZSetType().sameType(integral.getOutputIndexedZSetType())); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSimpleOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSimpleOperator.java index c06a1c9721..577d15a131 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSimpleOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSimpleOperator.java @@ -49,19 +49,15 @@ public abstract class DBSPSimpleOperator extends DBSPOperator /** True if the output of the operator is a multiset. Conservative approximation; * if this is 'false', it is surely false. If it is true, the output may still be a set. */ public final boolean isMultiset; - /** True if the operator contains an integrator */ - public final boolean containsIntegrator; protected DBSPSimpleOperator(CalciteRelNode node, String operation, @Nullable DBSPExpression function, DBSPType outputType, - boolean isMultiset, @Nullable String comment, - boolean containsIntegrator) { + boolean isMultiset, @Nullable String comment) { super(node, comment); this.operation = operation; this.function = function; this.outputType = outputType; this.isMultiset = isMultiset; - this.containsIntegrator = containsIntegrator; if (!operation.startsWith("waterline") && !operation.startsWith("apply") && !operation.startsWith("transaction_delay") && @@ -102,9 +98,8 @@ public OutputPort outputPort() { public DBSPSimpleOperator(CalciteRelNode node, String operation, @Nullable DBSPExpression function, - DBSPType outputType, boolean isMultiset, - boolean containsIntegrator) { - this(node, operation, function, outputType, isMultiset, null, containsIntegrator); + DBSPType outputType, boolean isMultiset) { + this(node, operation, function, outputType, isMultiset, null); } public DBSPSimpleOperator copyAnnotations(DBSPSimpleOperator source) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSourceBaseOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSourceBaseOperator.java index d1e52ba654..ff66b3e5f0 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSourceBaseOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSourceBaseOperator.java @@ -52,7 +52,7 @@ public abstract class DBSPSourceBaseOperator protected DBSPSourceBaseOperator( CalciteRelNode node, String operation, DBSPType outputType, boolean isMultiset, ProgramIdentifier tableName, TableMetadata metadata, @Nullable String comment) { - super(node, operation, null, outputType, isMultiset, comment, false); + super(node, operation, null, outputType, isMultiset, comment); this.tableName = tableName; this.metadata = metadata; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSourceMapOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSourceMapOperator.java index 8341aabd3c..f6c44b1492 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSourceMapOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSourceMapOperator.java @@ -1,7 +1,6 @@ package org.dbsp.sqlCompiler.circuit.operator; import com.fasterxml.jackson.databind.JsonNode; -import org.dbsp.sqlCompiler.circuit.IInputMapOperator; import org.dbsp.sqlCompiler.circuit.OutputPort; import org.dbsp.sqlCompiler.compiler.TableMetadata; import org.dbsp.sqlCompiler.compiler.backend.JsonDecoder; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamAggregateOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamAggregateOperator.java index 2abc7d75d8..6cfb0250c4 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamAggregateOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamAggregateOperator.java @@ -41,14 +41,14 @@ import javax.annotation.Nullable; import java.util.List; -public final class DBSPStreamAggregateOperator extends DBSPAggregateOperatorBase { +public final class DBSPStreamAggregateOperator extends DBSPAggregateOperatorBase implements INonIncremental { public DBSPStreamAggregateOperator(CalciteRelNode node, DBSPTypeIndexedZSet outputType, @Nullable DBSPAggregator function, @Nullable DBSPAggregateList aggregateList, OutputPort input) { super(node, "stream_aggregate", - outputType, function, aggregateList, false, input, false); + outputType, function, aggregateList, false, input); Utilities.enforce(aggregateList == null || aggregateList.rowVar.getType().sameType(input.getOutputIndexedZSetType().elementType.ref())); Utilities.enforce(aggregateList == null || diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamAntiJoinOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamAntiJoinOperator.java index e53f0e0c01..23f3267c86 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamAntiJoinOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamAntiJoinOperator.java @@ -15,9 +15,9 @@ import java.util.List; /** Currently there is no corespondent operator in DBSP. */ -public final class DBSPStreamAntiJoinOperator extends DBSPBinaryOperator { +public final class DBSPStreamAntiJoinOperator extends DBSPBinaryOperator implements INonIncremental { public DBSPStreamAntiJoinOperator(CalciteRelNode node, OutputPort left, OutputPort right) { - super(node, "stream_antijoin", null, left.outputType(), left.isMultiset(), left, right, false); + super(node, "stream_antijoin", null, left.outputType(), left.isMultiset(), left, right); left.getOutputIndexedZSetType(); right.getOutputIndexedZSetType(); Utilities.enforce(left.getOutputIndexedZSetType().keyType.sameType(right.getOutputIndexedZSetType().keyType), diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamDistinctOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamDistinctOperator.java index 067cc52526..4835cb63a7 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamDistinctOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPStreamDistinctOperator.java @@ -36,9 +36,9 @@ import javax.annotation.Nullable; import java.util.List; -public final class DBSPStreamDistinctOperator extends DBSPUnaryOperator { +public final class DBSPStreamDistinctOperator extends DBSPUnaryOperator implements INonIncremental { public DBSPStreamDistinctOperator(CalciteRelNode node, OutputPort input) { - super(node, "stream_distinct", null, input.outputType(), false, input, false); + super(node, "stream_distinct", null, input.outputType(), false, input); } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSubtractOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSubtractOperator.java index feb7cc3b60..9d6041b554 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSubtractOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSubtractOperator.java @@ -37,9 +37,9 @@ import javax.annotation.Nullable; import java.util.List; -public final class DBSPSubtractOperator extends DBSPBinaryOperator { +public final class DBSPSubtractOperator extends DBSPBinaryOperator implements ILinear { public DBSPSubtractOperator(CalciteRelNode node, OutputPort left, OutputPort right) { - super(node, "minus", null, left.outputType(), false, left, right, false); + super(node, "minus", null, left.outputType(), false, left, right); if (!left.outputType().sameType(right.outputType())) throw new InternalCompilerError("Inputs do not have the same type " + left.outputType() + " and " + right.outputType(), this); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSumOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSumOperator.java index edf11d76f8..eea3ce88bd 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSumOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPSumOperator.java @@ -38,9 +38,9 @@ import javax.annotation.Nullable; import java.util.List; -public final class DBSPSumOperator extends DBSPSimpleOperator { +public final class DBSPSumOperator extends DBSPSimpleOperator implements ILinear { public DBSPSumOperator(CalciteRelNode node, List inputs) { - super(node, "sum", null, inputs.get(0).outputType(), true, false); + super(node, "sum", null, inputs.get(0).outputType(), true); for (OutputPort op: inputs) { this.addInput(op); if (!op.outputType().sameType(this.outputType)) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPUnaryOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPUnaryOperator.java index 1932bbe172..6d66f0fc5c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPUnaryOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPUnaryOperator.java @@ -32,26 +32,18 @@ /** Base class for all DBSP query operators that have a single input. */ public abstract class DBSPUnaryOperator extends DBSPSimpleOperator { - protected DBSPUnaryOperator(CalciteRelNode node, String operation, - @Nullable DBSPExpression function, DBSPType outputType, - boolean isMultiset, OutputPort source, - boolean containsIntegrator) { - this(node, operation, function, outputType, isMultiset, source, null, containsIntegrator); - } - protected DBSPUnaryOperator(CalciteRelNode node, String operation, @Nullable DBSPExpression function, DBSPType outputType, boolean isMultiset, OutputPort source) { - this(node, operation, function, outputType, isMultiset, source, null, false); + this(node, operation, function, outputType, isMultiset, source, null); } @SuppressWarnings("SameParameterValue") protected DBSPUnaryOperator(CalciteRelNode node, String operation, @Nullable DBSPExpression function, DBSPType outputType, boolean isMultiset, OutputPort source, - @Nullable String comment, - boolean containsIntegrator) { - super(node, operation, function, outputType, isMultiset, comment, containsIntegrator); + @Nullable String comment) { + super(node, operation, function, outputType, isMultiset, comment); this.addInput(source); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPUpsertFeedbackOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPUpsertFeedbackOperator.java index 8fa1fcac7a..c3872c8e3d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPUpsertFeedbackOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPUpsertFeedbackOperator.java @@ -19,7 +19,7 @@ * to the indexed collection and produces a corresponding retraction * for the pre-existing key. */ @NonCoreIR -public final class DBSPUpsertFeedbackOperator extends DBSPUnaryOperator { +public final class DBSPUpsertFeedbackOperator extends DBSPUnaryOperator implements IContainsIntegrator { public DBSPUpsertFeedbackOperator(CalciteRelNode node, OutputPort source) { super(node, "upsert_feedback", null, source.outputType(), source.isMultiset(), source); source.getOutputIndexedZSetType(); // asserts that the type is right diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPViewBaseOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPViewBaseOperator.java index aa471dc0f6..48aeb52f12 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPViewBaseOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPViewBaseOperator.java @@ -27,7 +27,7 @@ protected DBSPViewBaseOperator( CalciteRelNode node, String operation, @Nullable DBSPExpression function, ProgramIdentifier viewName, String query, DBSPType originalRowType, ViewMetadata metadata, OutputPort input) { - super(node, operation, function, input.outputType(), input.isMultiset(), input, false); + super(node, operation, function, input.outputType(), input.isMultiset(), input); this.metadata = metadata; this.query = query; this.viewName = viewName; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWaterlineOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWaterlineOperator.java index 93235aeadc..dec1d0bcde 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWaterlineOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWaterlineOperator.java @@ -19,7 +19,7 @@ /** Given a stream, it computes function(extractTS(stream), delay(this, init)). * This operator is special: the output is replicated for all workers. * See the comments for {@link DBSPApplyOperator}. */ -public final class DBSPWaterlineOperator extends DBSPUnaryOperator { +public final class DBSPWaterlineOperator extends DBSPUnaryOperator implements IStateful { /** Initial value of waterline; a closure with 0 parameters */ public final DBSPClosureExpression init; /** Function which extracts a timestamp; a closure with two parameters; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWeighOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWeighOperator.java index 17b52b0a45..65fddb8e35 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWeighOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWeighOperator.java @@ -17,7 +17,7 @@ import java.util.List; @NonCoreIR -public final class DBSPWeighOperator extends DBSPUnaryOperator { +public final class DBSPWeighOperator extends DBSPUnaryOperator implements ILinear { static DBSPTypeZSet outputType(DBSPTypeIndexedZSet sourceType) { return new DBSPTypeZSet(sourceType.elementType); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWindowOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWindowOperator.java index c4494d54ae..f4027b9587 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWindowOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWindowOperator.java @@ -24,7 +24,7 @@ * elements in the left input are compared with the two scalars * in the pair; when they fall between the two limits, * they are emitted to the output ZSet. */ -public final class DBSPWindowOperator extends DBSPBinaryOperator { +public final class DBSPWindowOperator extends DBSPBinaryOperator implements IContainsIntegrator { public final boolean lowerInclusive; public final boolean upperInclusive; @@ -32,8 +32,8 @@ public DBSPWindowOperator( CalciteRelNode node, CalciteObject object, boolean lowerInclusive, boolean upperInclusive, OutputPort data, OutputPort control) { - super(node, "window", new NoExpression(object, DBSPTypeVoid.INSTANCE), data.outputType(), data.isMultiset(), - data, control, true); + super(node, "window", new NoExpression(object, DBSPTypeVoid.INSTANCE), data.outputType(), + data.isMultiset(), data, control); // Check that the left input and output are indexed ZSets this.getOutputIndexedZSetType(); this.lowerInclusive = lowerInclusive; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IContainsIntegrator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IContainsIntegrator.java new file mode 100644 index 0000000000..59267716d5 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IContainsIntegrator.java @@ -0,0 +1,4 @@ +package org.dbsp.sqlCompiler.circuit.operator; + +/** Interface implemented by all operators which contain one or more integrators */ +public interface IContainsIntegrator extends IStateful {} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/GCOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IGCOperator.java similarity index 79% rename from sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/GCOperator.java rename to sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IGCOperator.java index bce53261ef..372aa2fee6 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/GCOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IGCOperator.java @@ -3,6 +3,6 @@ import org.dbsp.sqlCompiler.ir.IDBSPOuterNode; /** Interface implemented by operators that perform Garbage collection. */ -public interface GCOperator extends IDBSPOuterNode { +public interface IGCOperator extends IDBSPOuterNode { DBSPSimpleOperator asOperator(); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/IInputMapOperator.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IInputMapOperator.java similarity index 96% rename from sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/IInputMapOperator.java rename to sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IInputMapOperator.java index 83f0a57bff..8d63a54f55 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/IInputMapOperator.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IInputMapOperator.java @@ -1,7 +1,5 @@ -package org.dbsp.sqlCompiler.circuit; +package org.dbsp.sqlCompiler.circuit.operator; -import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; -import org.dbsp.sqlCompiler.circuit.operator.IInputOperator; import org.dbsp.sqlCompiler.compiler.TableMetadata; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.ProgramIdentifier; import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IJoin.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IJoin.java new file mode 100644 index 0000000000..7babb9e141 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IJoin.java @@ -0,0 +1,4 @@ +package org.dbsp.sqlCompiler.circuit.operator; + +/** Interface implemented by all join (and antijoin) operators */ +public interface IJoin extends IContainsIntegrator {} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/ILinear.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/ILinear.java new file mode 100644 index 0000000000..ae4cbc7337 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/ILinear.java @@ -0,0 +1,4 @@ +package org.dbsp.sqlCompiler.circuit.operator; + +/** Interface implemented by linear operators, including linear aggregates */ +public interface ILinear {} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/ILinearAggregate.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/ILinearAggregate.java new file mode 100644 index 0000000000..f5776806c7 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/ILinearAggregate.java @@ -0,0 +1,4 @@ +package org.dbsp.sqlCompiler.circuit.operator; + +/** Interface implemented by linear aggregate operators */ +public interface ILinearAggregate extends ILinear, IContainsIntegrator {} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/IMultiOutput.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IMultiOutput.java similarity index 69% rename from sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/IMultiOutput.java rename to sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IMultiOutput.java index 2ceef5ccd7..3a3a010314 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/IMultiOutput.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IMultiOutput.java @@ -1,6 +1,6 @@ -package org.dbsp.sqlCompiler.circuit; +package org.dbsp.sqlCompiler.circuit.operator; -import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; +import org.dbsp.sqlCompiler.circuit.OutputPort; /** Interface implemented by operators with multiple outputs */ public interface IMultiOutput { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/INonIncremental.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/INonIncremental.java new file mode 100644 index 0000000000..0539c51cc0 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/INonIncremental.java @@ -0,0 +1,4 @@ +package org.dbsp.sqlCompiler.circuit.operator; + +/** A non-incremental operator; shold not appear in incremental circuits, but appears in testing circuits. */ +public interface INonIncremental {} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/INonLinearAggregate.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/INonLinearAggregate.java new file mode 100644 index 0000000000..644e30c8d5 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/INonLinearAggregate.java @@ -0,0 +1,3 @@ +package org.dbsp.sqlCompiler.circuit.operator; + +public interface INonLinearAggregate extends IContainsIntegrator {} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IStateful.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IStateful.java new file mode 100644 index 0000000000..04b0582acd --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/IStateful.java @@ -0,0 +1,5 @@ +package org.dbsp.sqlCompiler.circuit.operator; + +/** Interface implemented by all operators that contain state. + * e.g., delays are stateful, but do not contain integrators. */ +public interface IStateful {} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/dot/ToDotNodesVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/dot/ToDotNodesVisitor.java index 7bb8777be5..41577dab1e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/dot/ToDotNodesVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/dot/ToDotNodesVisitor.java @@ -31,7 +31,8 @@ import org.dbsp.sqlCompiler.ir.expression.DBSPFlatmap; import org.dbsp.util.IndentStream; -/** Visitor which emits the circuit nodes in a graphviz file */ +/** Visitor which emits the circuit nodes in a graphviz file. + * The compiler options control the detail. On verbosity=0 table and view names are ommitted. */ public class ToDotNodesVisitor extends CircuitVisitor { protected final IndentStream stream; // A higher value -> more details @@ -67,7 +68,7 @@ public void endVisit() { @Override public VisitDecision preorder(DBSPSourceBaseOperator node) { - String name = node.tableName + " " + node.operation; + String name = (this.compiler.options.ioOptions.verbosity > 0 ? (node.tableName + " ") : "") + node.operation; this.stream.append(node.getNodeName(false)) .append(" [ shape=box style=filled fillcolor=lightgrey label=\"") .append(node.getIdString()) @@ -104,8 +105,7 @@ public VisitDecision preorder(DBSPViewBaseOperator node) { .append(node.getIdString()) .append(isMultiset(node)) .append(annotations(node)) - .append(" ") - .append(node.viewName.name()) + .append(this.compiler.options.ioOptions.verbosity > 0 ? " " + node.viewName.name() : "") .append("\"") .append(" style=filled fillcolor=lightgrey") .append("]") diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java index fb5e3e58f3..e003f2fbbe 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustVisitor.java @@ -26,7 +26,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; -import org.dbsp.sqlCompiler.circuit.IInputMapOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPIntegrateOperator; +import org.dbsp.sqlCompiler.circuit.operator.IContainsIntegrator; +import org.dbsp.sqlCompiler.circuit.operator.IInputMapOperator; import org.dbsp.sqlCompiler.circuit.OutputPort; import org.dbsp.sqlCompiler.circuit.annotation.OperatorHash; import org.dbsp.sqlCompiler.circuit.annotation.Recursive; @@ -506,9 +508,10 @@ void generateNestedStructs(IDBSPInnerNode node, boolean global) { /** Emit the call to the function associated with the operator, including the open parens */ void operationCall(DBSPSimpleOperator operator) { this.builder.append(operator.operation); - if (operator.containsIntegrator && + if (operator.is(IContainsIntegrator.class) && !operator.is(DBSPJoinBaseOperator.class) && - !operator.is(DBSPAntiJoinOperator.class)) { + !operator.is(DBSPAntiJoinOperator.class) && + !operator.is(DBSPIntegrateOperator.class)) { this.builder.append("_persistent(hash, "); } else { this.builder.append("("); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java index 1e823e5802..5db2d61ad5 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/multi/SingleOperatorWriter.java @@ -13,7 +13,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceMultisetOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSumOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPViewBaseOperator; -import org.dbsp.sqlCompiler.circuit.operator.GCOperator; +import org.dbsp.sqlCompiler.circuit.operator.IGCOperator; import org.dbsp.sqlCompiler.circuit.operator.IInputOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.backend.rust.BaseRustCodeGenerator; @@ -96,7 +96,7 @@ public void write(DBSPCompiler compiler) { this.builder().append("use ").append(dep).append("::*;").newline(); boolean hasOutput = (!operator.is(DBSPViewBaseOperator.class) || useHandles) && - !operator.is(GCOperator.class) && + !operator.is(IGCOperator.class) && !operator.is(DBSPInternOperator.class); this.builder().append("pub fn create_") .append(name) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/BalancedJoins.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/BalancedJoins.java index 35bb52e882..727c1d606b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/BalancedJoins.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/BalancedJoins.java @@ -11,7 +11,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPLeftJoinOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPNestedOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; -import org.dbsp.sqlCompiler.circuit.operator.GCOperator; +import org.dbsp.sqlCompiler.circuit.operator.IGCOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.util.Linq; import org.dbsp.util.graph.Port; @@ -26,7 +26,7 @@ protected BalancedJoins(DBSPCompiler compiler, CircuitGraphs graphs) { private boolean hasGcSuccessor(DBSPOperator operator) { for (Port succ: this.getGraph().getSuccessors(operator)) { - if (succ.node().is(GCOperator.class)) + if (succ.node().is(IGCOperator.class)) // only input 0 of these operators affects the GC return succ.port() == 0; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CSE.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CSE.java index cbe4a6992c..cb80146c2d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CSE.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CSE.java @@ -1,14 +1,14 @@ package org.dbsp.sqlCompiler.compiler.visitors.outer; import org.dbsp.sqlCompiler.circuit.ICircuit; -import org.dbsp.sqlCompiler.circuit.IMultiOutput; +import org.dbsp.sqlCompiler.circuit.operator.IMultiOutput; import org.dbsp.sqlCompiler.circuit.operator.DBSPConstantOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPDeltaOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPNestedOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; import org.dbsp.sqlCompiler.circuit.OutputPort; -import org.dbsp.sqlCompiler.circuit.operator.GCOperator; +import org.dbsp.sqlCompiler.circuit.operator.IGCOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.util.Logger; import org.dbsp.util.graph.Port; @@ -68,7 +68,7 @@ public void postorder(DBSPConstantOperator operator) { boolean hasGcSuccessor(DBSPOperator operator) { for (Port succ: this.getGraph().getSuccessors(operator)) { - if (succ.node().is(GCOperator.class)) + if (succ.node().is(IGCOperator.class)) // only input 0 of these operators affects the GC return succ.port() == 0; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitCloneVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitCloneVisitor.java index 2d2a384391..a19d712a11 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitCloneVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitCloneVisitor.java @@ -26,7 +26,7 @@ import org.dbsp.sqlCompiler.circuit.DBSPDeclaration; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; import org.dbsp.sqlCompiler.circuit.ICircuit; -import org.dbsp.sqlCompiler.circuit.IMultiOutput; +import org.dbsp.sqlCompiler.circuit.operator.IMultiOutput; import org.dbsp.sqlCompiler.circuit.OutputPort; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.ICompilerComponent; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java index c8bc2a9e08..4a4803f603 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java @@ -164,6 +164,7 @@ void createOptimizer() { this.add(new CompactNames(compiler)); this.add(new MerkleOuter(compiler, true)); this.add(new MerkleOuter(compiler, false)); + this.add(new CircuitStatistics(compiler)); } public DBSPCircuit optimize(DBSPCircuit input) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitRewriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitRewriter.java index 0fdc99228c..5628d30cca 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitRewriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitRewriter.java @@ -23,7 +23,7 @@ package org.dbsp.sqlCompiler.compiler.visitors.outer; -import org.dbsp.sqlCompiler.circuit.IMultiOutput; +import org.dbsp.sqlCompiler.circuit.operator.IMultiOutput; import org.dbsp.sqlCompiler.circuit.operator.DBSPAsofJoinOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPAggregateLinearPostprocessOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPAggregateLinearPostprocessRetainKeysOperator; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitStatistics.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitStatistics.java new file mode 100644 index 0000000000..8df3f3ec80 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitStatistics.java @@ -0,0 +1,135 @@ +package org.dbsp.sqlCompiler.compiler.visitors.outer; + +import org.dbsp.sqlCompiler.circuit.OutputPort; +import org.dbsp.sqlCompiler.circuit.operator.DBSPNestedOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPViewBaseOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPViewDeclarationOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPWindowOperator; +import org.dbsp.sqlCompiler.circuit.operator.IGCOperator; +import org.dbsp.sqlCompiler.circuit.operator.IInputOperator; +import org.dbsp.sqlCompiler.circuit.operator.IJoin; +import org.dbsp.sqlCompiler.circuit.operator.ILinear; +import org.dbsp.sqlCompiler.circuit.operator.ILinearAggregate; +import org.dbsp.sqlCompiler.circuit.operator.INonIncremental; +import org.dbsp.sqlCompiler.circuit.operator.INonLinearAggregate; +import org.dbsp.sqlCompiler.circuit.operator.IStateful; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; +import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeIndexedZSet; +import org.dbsp.util.Logger; +import org.dbsp.util.Utilities; + +import java.util.HashMap; +import java.util.Map; + +/** Compute some simple circuit statistics. To display the statistics the compiler can be invoked + * with the following option: "-TCircuitStatistics=1" */ +public class CircuitStatistics extends CircuitVisitor { + static class Statistics { + public int totalOperators; + public int tables; + public int views; + public int joins; + public int linear; + public int linearAggregates; + public int nonLinearAggregates; + public int windows; + public int gcOperators; + public int nonIncremental; + public int nested; + public int maxTupleWidth; + public int maxDepth; + + @Override + public String toString() { + return "{" + + "\n \"totalOperators\": " + totalOperators + + ",\n \"gcOperators\": " + gcOperators + + ",\n \"tables\": " + tables + + ",\n \"views\": " + views + + ",\n \"joins\": " + joins + + ",\n \"linear\": " + linear + + ",\n \"linearAggregates\": " + linearAggregates + + ",\n \"nonLinearAggregates\": " + nonLinearAggregates + + ",\n \"nonIncremental\": " + nonIncremental + + ",\n \"windows\": " + windows + + ",\n \"nested\": " + nested + + ",\n \"stateful\": " + stateful + + ",\n \"maxTupleWidth\": " + maxTupleWidth + + "\n \"maxDepth\": " + maxDepth + + "\n}"; + } + + public int stateful; + + void updateWidth(int width) { + this.maxTupleWidth = Math.max(width, this.maxTupleWidth); + } + + public void updateDepth(int depth) { + this.maxDepth = Math.max(depth, this.maxDepth); + } + }; + + final Statistics stats; + final Map depth; + + public CircuitStatistics(DBSPCompiler compiler) { + super(compiler); + this.stats = new Statistics(); + this.depth = new HashMap<>(); + } + + public int tupleWidth(DBSPType type) { + if (type.is(DBSPTypeIndexedZSet.class)) { + var ix = type.to(DBSPTypeIndexedZSet.class); + return ix.keyType.getToplevelFieldCount() + ix.elementType.getToplevelFieldCount(); + } + return type.getToplevelFieldCount(); + } + + @Override + public void postorder(DBSPOperator operator) { + this.stats.totalOperators++; + if (operator.is(IInputOperator.class) && !operator.is(DBSPViewDeclarationOperator.class)) + this.stats.tables++; + if (operator.is(DBSPViewBaseOperator.class)) + this.stats.views++; + if (operator.is(IGCOperator.class)) + this.stats.gcOperators++; + if (operator.is(DBSPWindowOperator.class)) + this.stats.windows++; + if (operator.is(IJoin.class)) + this.stats.joins++; + if (operator.is(ILinearAggregate.class)) + this.stats.linearAggregates++; + if (operator.is(ILinear.class) && !operator.is(ILinearAggregate.class)) + this.stats.linear++; + if (operator.is(IStateful.class)) + this.stats.stateful++; + if (operator.is(INonLinearAggregate.class)) + this.stats.nonLinearAggregates++; + if (operator.is(DBSPNestedOperator.class)) + this.stats.nested++; + if (operator.is(INonIncremental.class)) + this.stats.nonIncremental++; + int depth = 0; + for (OutputPort port: operator.inputs) { + int width = this.tupleWidth(port.outputType()); + this.stats.updateWidth(width); + int inputDepth = this.depth.get(port.operator); + depth = Math.max(inputDepth + 1, depth); + } + Utilities.putNew(this.depth, operator, depth); + this.stats.updateDepth(depth); + } + + @Override + public void endVisit() { + Logger.INSTANCE.belowLevel(this, 1) + .appendSupplier(this.stats::toString) + .append("\n"); + super.endVisit(); + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerAsof.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerAsof.java index 2396ad1143..25c534a243 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerAsof.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerAsof.java @@ -9,7 +9,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPMapIndexOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; -import org.dbsp.sqlCompiler.circuit.operator.GCOperator; +import org.dbsp.sqlCompiler.circuit.operator.IGCOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.frontend.TypeCompiler; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteRelNode; @@ -52,10 +52,10 @@ public class LowerAsof implements CircuitTransform { @Override public DBSPCircuit apply(DBSPCircuit circuit) { // Maps an integrate trace node in the original graph to the asof join operator that it should be moved to - Map leftGces = new HashMap<>(); - Map rightGces = new HashMap<>(); + Map leftGces = new HashMap<>(); + Map rightGces = new HashMap<>(); // Maps an integrate trace operator in the new graph to the original one - Map original = new HashMap<>(); + Map original = new HashMap<>(); Graph graph = new Graph(this.compiler); graph.apply(circuit); @@ -102,15 +102,15 @@ public String toString() { /** Move the integrate trace operator to the new join */ static class MoveGC extends CircuitCloneVisitor { - final Map leftGces; - final Map rightGces; - final Map original; + final Map leftGces; + final Map rightGces; + final Map original; public MoveGC( DBSPCompiler compiler, - Map leftGces, - Map rightGces, - Map original) { + Map leftGces, + Map rightGces, + Map original) { super(compiler, false); this.leftGces = leftGces; this.rightGces = rightGces; @@ -123,7 +123,7 @@ public void postorder(DBSPIntegrateTraceRetainValuesOperator operator) { // Because we only replace such operators in this visitor, and such // operators have no successors, we know that the graph will not change // anywhere else, including the ConcreteAsofJoinOperator - GCOperator original = this.original.get(operator); + IGCOperator original = this.original.get(operator); if (original == null || !this.leftGces.containsKey(original)) { super.postorder(operator); return; @@ -154,7 +154,7 @@ public void postorder(DBSPIntegrateTraceRetainNValuesOperator operator) { // Because we only replace such operators in this visitor, and such // operators have no successors, we know that the graph will not change // anywhere else, including the ConcreteAsofJoinOperator - GCOperator original = this.original.get(operator); + IGCOperator original = this.original.get(operator); if (original == null || !this.rightGces.containsKey(original)) { super.postorder(operator); return; @@ -183,15 +183,15 @@ public void postorder(DBSPIntegrateTraceRetainNValuesOperator operator) { } static class LowerAsofInner extends CircuitCloneWithGraphsVisitor { - final Map leftGces; - final Map rightGces; - final Map original; + final Map leftGces; + final Map rightGces; + final Map original; public LowerAsofInner( DBSPCompiler compiler, CircuitGraphs graphs, - Map leftGces, - Map rightGces, - Map original) { + Map leftGces, + Map rightGces, + Map original) { super(compiler, graphs, false); this.leftGces = leftGces; this.rightGces = rightGces; @@ -297,14 +297,14 @@ public void postorder(DBSPAsofJoinOperator join) { // If there is an IntegrateTraceRetainValues operator on the left input, it needs to be moved. CircuitGraph graph = this.getGraph(); for (Port port : graph.getSuccessors(join.left().operator)) { - GCOperator op = port.node().as(DBSPIntegrateTraceRetainValuesOperator.class); + IGCOperator op = port.node().as(DBSPIntegrateTraceRetainValuesOperator.class); if (op != null) Utilities.putNew(this.leftGces, op, result); } // Similar procedure for the right input for (Port port : graph.getSuccessors(join.right().operator)) { - GCOperator op = port.node().as(DBSPIntegrateTraceRetainNValuesOperator.class); + IGCOperator op = port.node().as(DBSPIntegrateTraceRetainNValuesOperator.class); if (op != null) Utilities.putNew(this.rightGces, op, result); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/RemoveNoops.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/RemoveNoops.java index 276de862b5..0d26280cb5 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/RemoveNoops.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/RemoveNoops.java @@ -2,7 +2,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPNoopOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; -import org.dbsp.sqlCompiler.circuit.operator.GCOperator; +import org.dbsp.sqlCompiler.circuit.operator.IGCOperator; import org.dbsp.sqlCompiler.circuit.OutputPort; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.util.Linq; @@ -19,7 +19,7 @@ public RemoveNoops(DBSPCompiler compiler, CircuitGraphs graphs) { @Override public void postorder(DBSPNoopOperator operator) { List> destinations = this.getGraph().getSuccessors(operator); - boolean keep = Linq.any(destinations, d -> d.node().is(GCOperator.class)); + boolean keep = Linq.any(destinations, d -> d.node().is(IGCOperator.class)); if (keep) { super.postorder(operator); } else { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/StrayGC.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/StrayGC.java index 89f0302b56..d170edb8fe 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/StrayGC.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/StrayGC.java @@ -11,6 +11,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPLagOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; +import org.dbsp.sqlCompiler.circuit.operator.IContainsIntegrator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError; import org.dbsp.util.graph.Port; @@ -31,7 +32,7 @@ void check(DBSPBinaryOperator operator) { DBSPOperator so = sibling.node(); if (so.is(DBSPSimpleOperator.class)) { DBSPSimpleOperator simple = so.to(DBSPSimpleOperator.class); - if (simple.containsIntegrator) { + if (simple.is(IContainsIntegrator.class)) { return; } } else { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/MergeGC.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/MergeGC.java index 5542242cec..dca173f46e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/MergeGC.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/MergeGC.java @@ -3,7 +3,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPNoopOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator; -import org.dbsp.sqlCompiler.circuit.operator.GCOperator; +import org.dbsp.sqlCompiler.circuit.operator.IGCOperator; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.visitors.outer.CSE; import org.dbsp.sqlCompiler.compiler.visitors.outer.CircuitGraphs; @@ -48,7 +48,7 @@ DBSPSimpleOperator getSingleGcSuccessor(DBSPOperator operator) { return null; List> baseDests = Linq.where( this.getGraph().getSuccessors(operator), - p -> (p.node().is(GCOperator.class) && p.port() == 0)); + p -> (p.node().is(IGCOperator.class) && p.port() == 0)); if (baseDests.size() != 1) return null; Port port = baseDests.get(0);