Thanks to visit codestin.com
Credit goes to github.com

Skip to content

[SQL] Fix several compiler bugs discovered while testing more complex programs #5478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mihaibudiu merged 1 commit into from
Jan 20, 2026
Merged

[SQL] Fix several compiler bugs discovered while testing more complex programs #5478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .../SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/ToJsonInnerVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI64Literal;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI8Literal;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPISizeLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPInternedStringLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMillisLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMonthsLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPKeywordLiteral;
Expand Down Expand Up @@ -330,6 +331,11 @@ public void postorder(DBSPISizeLiteral node) {
super.postorder(node);
}

@Override
public void postorder(DBSPInternedStringLiteral node) {
super.postorder(node);
}

@Override
public void postorder(DBSPKeywordLiteral node) {
this.property("keyword");
Expand Down
7 changes: 7 additions & 0 deletions ...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI64Literal;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI8Literal;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPISizeLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPInternedStringLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMillisLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMonthsLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPLiteral;
Expand Down Expand Up @@ -794,6 +795,12 @@ public VisitDecision preorder(DBSPISizeLiteral literal) {
return VisitDecision.STOP;
}

@Override
public VisitDecision preorder(DBSPInternedStringLiteral literal) {
this.builder.append("None");
return VisitDecision.STOP;
}

@Override
public VisitDecision preorder(DBSPMinMax aggregator) {
if (aggregator.postProcessing != null)
Expand Down
1 change: 1 addition & 0 deletions ...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,7 @@ public DBSPCompiler compiler() {
}

public DBSPType convertType(SourcePositionRange context, RelDataType dt, boolean asStruct) {
dt = TypeCompiler.removeDuplicateFields(dt);
return this.compiler.getTypeCompiler().convertType(context, dt, asStruct);
}

Expand Down
16 changes: 11 additions & 5 deletions ...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ public DBSPExpression visitCorrelVariable(RexCorrelVariable correlVariable) {
public DBSPExpression visitLiteral(RexLiteral literal) {
CalciteObject node = CalciteObject.create(this.context, literal);
try {
DBSPType type = this.typeCompiler.convertType(node.getPositionRange(), literal.getType(), true);
DBSPType type = this.typeCompiler.convertType(node.getPositionRange(), literal.getType(), false);
if (literal.isNull())
return DBSPLiteral.none(type);
switch (type.code) {
Expand Down Expand Up @@ -528,9 +528,14 @@ public static DBSPExpression makeBinaryExpression(
DBSPExpression tmp = left;
left = right;
right = tmp;
type = typeWithNull;
}
return new DBSPTimeAddSub(node, type, opcode, left, right);
// There seems to be a bug in Calcite's type inference which returns
// nullable results when subtracting non-nullable values. Correct for that.
DBSPExpression result = new DBSPTimeAddSub(node, typeWithNull, opcode, left, right);
if (!typeWithNull.sameType(type)) {
result = result.cast(node, type, DBSPCastExpression.CastType.SqlUnsafe);
}
return result;
}
}
if (leftType.is(IsTimeRelatedType.class) || rightType.is(IsTimeRelatedType.class))
Expand Down Expand Up @@ -848,9 +853,10 @@ DBSPExpression strictnessCheck(List<DBSPExpression> ops, DBSPType resultType) {
}

private boolean validateCast(DBSPType from, DBSPType to, boolean safe) {
if (from.is(DBSPTypeBinary.class) && to.is(IsDateType.class)) {
if (from.is(DBSPTypeBinary.class) && to.is(IsDateType.class))
return false;
}
if (from.is(DBSPTypeNull.class))
return true;
if (from.is(DBSPTypeBaseType.class) && to.is(DBSPTypeBaseType.class))
return true;
if (from.is(DBSPTypeVariant.class) || to.is(DBSPTypeVariant.class))
Expand Down
27 changes: 26 additions & 1 deletion ...piler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/TypeCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@

import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
import org.apache.calcite.rel.type.RelRecordType;
import org.apache.calcite.sql.type.SqlTypeName;
import org.dbsp.sqlCompiler.compiler.DBSPCompiler;
import org.dbsp.sqlCompiler.compiler.ICompilerComponent;
Expand Down Expand Up @@ -258,6 +260,29 @@ else if (li.getWidth() == ri.getWidth())
throw new UnimplementedException("Cast from " + right + " to " + left, left.getNode());
}

public static RelDataType removeDuplicateFields(RelDataType rowType) {
FreshName names = new FreshName(new HashSet<>());
List<RelDataTypeField> fields = new ArrayList<>();
boolean duplicates = false;
if (rowType.isStruct()) {
for (RelDataTypeField field: rowType.getFieldList()) {
String name = field.getName();
if (names.isUsed(name)) {
duplicates = true;
name = names.freshName(name, true);
fields.add(new RelDataTypeFieldImpl(name, field.getIndex(), field.getType()));
} else {
fields.add(field);
names.add(name);
}
}
}
if (!duplicates)
return rowType;
Utilities.enforce(rowType.getFieldCount() == fields.size());
return new RelRecordType(rowType.getStructKind(), fields, rowType.isNullable());
}

public DBSPType convertType(
CalciteObject node, ProgramIdentifier name,
List<RelColumnMetadata> columns, boolean asStruct, boolean mayBeNull) {
Expand Down Expand Up @@ -310,7 +335,7 @@ public DBSPType convertType(SourcePositionRange context, RelDataType dt, boolean
FreshName fieldNameGen = new FreshName(new HashSet<>());
int index = 0;
for (RelDataTypeField field : dt.getFieldList()) {
DBSPType type = this.convertType(context, field.getType(), true);
DBSPType type = this.convertType(context, field.getType(), asStruct);
String fieldName = field.getName();
if (this.compiler().options.languageOptions.lenient)
// If we are not lenient and names are duplicated
Expand Down
10 changes: 10 additions & 0 deletions ...piler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerRewriteVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI64Literal;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPISizeLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPInternedStringLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMillisLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMonthsLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPKeywordLiteral;
Expand Down Expand Up @@ -627,6 +628,15 @@ public VisitDecision preorder(DBSPISizeLiteral expression) {
return VisitDecision.STOP;
}

@Override
public VisitDecision preorder(DBSPInternedStringLiteral expression) {
this.push(expression);
this.pop(expression);
DBSPExpression result = new DBSPInternedStringLiteral();
this.map(expression, result);
return VisitDecision.STOP;
}

@Override
public VisitDecision preorder(DBSPKeywordLiteral expression) {
this.map(expression, expression);
Expand Down
9 changes: 9 additions & 0 deletions ...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI8Literal;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPISizeLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPInternedStringLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMillisLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMonthsLiteral;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPKeywordLiteral;
Expand Down Expand Up @@ -867,6 +868,10 @@ public VisitDecision preorder(DBSPISizeLiteral node) {
return this.preorder((DBSPLiteral) node);
}

public VisitDecision preorder(DBSPInternedStringLiteral node) {
return this.preorder((DBSPLiteral) node);
}

public VisitDecision preorder(DBSPGeoPointConstructor node) {
return this.preorder((DBSPExpression) node);
}
Expand Down Expand Up @@ -1517,6 +1522,10 @@ public void postorder(DBSPISizeLiteral node) {
this.postorder((DBSPLiteral) node);
}

public void postorder(DBSPInternedStringLiteral node) {
this.postorder((DBSPLiteral) node);
}

public void postorder(DBSPGeoPointConstructor node) {
this.postorder((DBSPExpression) node);
}
Expand Down
11 changes: 7 additions & 4 deletions ...r/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Projection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,12 @@ public class Projection extends InnerVisitor {

/** A pair containing an input (parameter) number (0, 1, 2, etc.)
* and an index field in the tuple of the corresponding input .*/
public record InputAndFieldIndex(int inputIndex, int fieldIndex) {}
public record InputAndFieldIndex(int inputIndex, int fieldIndex) {
@Override
public String toString() {
return this.inputIndex + "." + this.fieldIndex;
}
}

/**
* A list describing how each output of a projection is computed.
Expand Down Expand Up @@ -192,9 +197,7 @@ public VisitDecision preorder(DBSPVariablePath var) {
@Override
public VisitDecision preorder(DBSPCastExpression expression) {
DBSPType type = expression.getType();
if (!expression.source.getType().withMayBeNull(true)
.sameType(type.withMayBeNull(true)) ||
!this.allowNoopCasts) {
if (!expression.source.getType().sameTypeIgnoringNullability(type) || !this.allowNoopCasts) {
// A cast which only changes nullability is
// considered an identity function
return this.notProjection();
Expand Down
4 changes: 4 additions & 0 deletions ...er/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerAsof.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ public DBSPCircuit apply(DBSPCircuit circuit) {
var newJoin = leftGces.get(originalIntegrate);
if (newJoin != null)
cg.addEdge(newJoin, newIntegrate.asOperator(), 0);
newJoin = rightGces.get(originalIntegrate);
if (newJoin != null)
cg.addEdge(newJoin, newIntegrate.asOperator(), 0);
}
circuit.resort(cg);

Expand Down Expand Up @@ -299,6 +302,7 @@ public void postorder(DBSPAsofJoinOperator join) {
Utilities.putNew(this.leftGces, op, result);
}

// Similar procedure for the right input
for (Port<DBSPOperator> port : graph.getSuccessors(join.right().operator)) {
GCOperator op = port.node().as(DBSPIntegrateTraceRetainValuesLastNOperator.class);
if (op != null)
Expand Down
26 changes: 13 additions & 13 deletions ...src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/expansion/ExpandOperators.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,19 +410,6 @@ public void postorder(DBSPJoinFilterMapOperator operator) {
@Nullable DBSPFilterOperator rightFilter = null;
List<OutputPort> sumInputs = new ArrayList<>();

if (hasLeftIntegrator) {
leftIntegrator = new DBSPDelayedIntegralOperator(operator.getRelNode(), inputs.get(0));
this.addOperator(leftIntegrator);
leftIntegrator.copyAnnotations(operator.left().node());
rightJoin = new DBSPStreamJoinOperator(operator.getRelNode(), operator.getOutputZSetType(),
operator.getFunction(), operator.isMultiset, leftIntegrator.outputPort(), inputs.get(1),
operator.balanced);
this.addOperator(rightJoin);
rightFilter = new DBSPFilterOperator(operator.getRelNode(), operator.getFilter(), rightJoin.outputPort());
this.addOperator(rightFilter);
sumInputs.add(rightFilter.outputPort());
}

if (hasRightIntegrator) {
rightIntegrator = new DBSPDelayedIntegralOperator(operator.getRelNode(), inputs.get(1));
this.addOperator(rightIntegrator);
Expand All @@ -444,6 +431,19 @@ public void postorder(DBSPJoinFilterMapOperator operator) {
this.addOperator(filter);
sumInputs.add(filter.outputPort());

if (hasLeftIntegrator) {
leftIntegrator = new DBSPDelayedIntegralOperator(operator.getRelNode(), inputs.get(0));
this.addOperator(leftIntegrator);
leftIntegrator.copyAnnotations(operator.left().node());
rightJoin = new DBSPStreamJoinOperator(operator.getRelNode(), operator.getOutputZSetType(),
operator.getFunction(), operator.isMultiset, leftIntegrator.outputPort(), inputs.get(1),
operator.balanced);
this.addOperator(rightJoin);
rightFilter = new DBSPFilterOperator(operator.getRelNode(), operator.getFilter(), rightJoin.outputPort());
this.addOperator(rightFilter);
sumInputs.add(rightFilter.outputPort());
}

DBSPSumOperator sum = new DBSPSumOperator(operator.getRelNode(), sumInputs);
this.map(operator, sum);
this.addExpansion(operator, new JoinFilterMapExpansion(leftIntegrator, rightIntegrator,
Expand Down
21 changes: 19 additions & 2 deletions ...mpiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/intern/InternInner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ Pair<DBSPExpression, DBSPExpression> uninternBothOrNone(
DBSPType originalLeftType, DBSPType originalRightType) {
DBSPTypeCode lCode = left.getType().code;
DBSPTypeCode rCode = right.getType().code;
boolean mayBeNull = left.getType().mayBeNull;
if (lCode == DBSPTypeCode.TUPLE || lCode == DBSPTypeCode.RAW_TUPLE) {
Utilities.enforce(lCode == rCode);
DBSPTypeTupleBase lTuple = originalLeftType.to(DBSPTypeTupleBase.class);
Expand All @@ -359,12 +360,28 @@ Pair<DBSPExpression, DBSPExpression> uninternBothOrNone(
DBSPExpression[] rExpr = new DBSPExpression[lTuple.size()];
for (int i = 0; i < lTuple.size(); i++) {
var fields = this.uninternBothOrNone(
left.field(i).simplify(), right.field(i).simplify(),
left.unwrapIfNullable("Cannot be NULL").field(i).simplify(),
right.unwrapIfNullable("Cannot be NULL").field(i).simplify(),
lTuple.getFieldType(i), rTuple.getFieldType(i));
lExpr[i] = fields.left.applyCloneIfNeeded();
rExpr[i] = fields.right.applyCloneIfNeeded();
}
return Pair.of(lTuple.makeTuple(lExpr), lTuple.makeTuple(rExpr));
DBSPExpression leftResult;
DBSPExpression rightResult;
if (lCode == DBSPTypeCode.TUPLE) {
leftResult = new DBSPTupleExpression(mayBeNull, lExpr);
rightResult = new DBSPTupleExpression(mayBeNull, rExpr);
} else {
leftResult = new DBSPRawTupleExpression(lExpr);
rightResult = new DBSPRawTupleExpression(rExpr);
}
if (mayBeNull) {
leftResult = new DBSPIfExpression(left.getNode(), left.is_null(),
leftResult.getType().nullValue(), leftResult).simplify();
rightResult = new DBSPIfExpression(right.getNode(), right.is_null(),
rightResult.getType().nullValue(), rightResult).simplify();
}
return Pair.of(leftResult, rightResult);
}
if (lCode == DBSPTypeCode.INTERNED_STRING && rCode != DBSPTypeCode.INTERNED_STRING) {
left = callUnintern(left, originalLeftType);
Expand Down
14 changes: 1 addition & 13 deletions sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/IDBSPNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,4 @@
* An IR node that is used to represent DBSP circuits.
*/
@SuppressWarnings("unused")
public interface IDBSPNode extends ICastable, IHasId, ToIndentableString, IHasCalciteObject {
default <T> T checkNull(@Nullable T value) {
if (value == null)
this.error("Null pointer");
if (value == null)
throw new InternalCompilerError("Did not expect a null value", this);
return value;
}

default void error(String message) {
throw new InternalCompilerError(message, this.getNode());
}
}
public interface IDBSPNode extends ICastable, IHasId, ToIndentableString, IHasCalciteObject { }
13 changes: 13 additions & 0 deletions ...piler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPIfExpression.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,12 @@
import org.dbsp.sqlCompiler.compiler.visitors.inner.EquivalenceContext;
import org.dbsp.sqlCompiler.ir.IDBSPInnerNode;
import org.dbsp.sqlCompiler.compiler.visitors.inner.InnerVisitor;
import org.dbsp.sqlCompiler.ir.expression.literal.DBSPBoolLiteral;
import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool;
import org.dbsp.util.IIndentStream;

import javax.annotation.Nullable;
import java.util.Objects;

public final class DBSPIfExpression extends DBSPExpression {
public final DBSPExpression condition;
Expand Down Expand Up @@ -88,6 +90,17 @@ public boolean sameFields(IDBSPInnerNode other) {
this.negative == o.negative;
}

public DBSPExpression simplify() {
if (this.condition.is(DBSPBoolLiteral.class)) {
Boolean cond = this.condition.to(DBSPBoolLiteral.class).value;
if (cond == null || !cond) {
return Objects.requireNonNull(this.negative);
}
return this.positive;
}
return this;
}
Comment on lines +93 to +102
Copy link

Copilot AI Jan 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The simplify() method has a potential bug when handling null or false conditions. When the condition is null or false, the method returns Objects.requireNonNull(this.negative), but this.negative can be null (as indicated by the @nullable annotation). This will throw a NullPointerException when negative is null. The method should check if this.negative is null before returning it, or handle the case appropriately.

Copilot uses AI. Check for mistakes.

@Override
public IIndentStream toString(IIndentStream builder) {
builder.append("if ")
Expand Down
Loading
Loading