diff --git a/WORKSPACE b/WORKSPACE index 2c9bc18a2..d6e54d6ea 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -78,7 +78,7 @@ maven_install( "com.google.guava:guava-testlib:33.3.1-jre", "com.google.protobuf:protobuf-java:4.28.3", "com.google.protobuf:protobuf-java-util:4.28.3", - "com.google.re2j:re2j:1.7", + "com.google.re2j:re2j:1.8", "com.google.testparameterinjector:test-parameter-injector:1.18", "com.google.truth.extensions:truth-java8-extension:1.4.4", "com.google.truth.extensions:truth-proto-extension:1.4.4", diff --git a/bundle/src/test/java/dev/cel/bundle/CelImplTest.java b/bundle/src/test/java/dev/cel/bundle/CelImplTest.java index db4b60fe1..a5a8d7db1 100644 --- a/bundle/src/test/java/dev/cel/bundle/CelImplTest.java +++ b/bundle/src/test/java/dev/cel/bundle/CelImplTest.java @@ -2032,6 +2032,39 @@ public void program_comprehensionDisabled_throws() throws Exception { assertThat(e.getErrorCode()).isEqualTo(CelErrorCode.ITERATION_BUDGET_EXCEEDED); } + @Test + public void program_regexProgramSizeUnderLimit_success() throws Exception { + Cel cel = + standardCelBuilderWithMacros() + .setOptions(CelOptions.current().maxRegexProgramSize(7).build()) + .build(); + // See + // https://github.com/google/re2j/blob/84237cbbd0fbd637c6eb6856717c1e248daae729/javatests/com/google/re2j/PatternTest.java#L175 for program size + CelAbstractSyntaxTree ast = cel.compile("'foo'.matches('(a+b)')").getAst(); + + assertThat(cel.createProgram(ast).eval()).isEqualTo(false); + } + + @Test + public void program_regexProgramSizeExceedsLimit_throws() throws Exception { + Cel cel = + standardCelBuilderWithMacros() + .setOptions(CelOptions.current().maxRegexProgramSize(6).build()) + .build(); + // See + // https://github.com/google/re2j/blob/84237cbbd0fbd637c6eb6856717c1e248daae729/javatests/com/google/re2j/PatternTest.java#L175 for program size + CelAbstractSyntaxTree ast = cel.compile("'foo'.matches('(a+b)')").getAst(); + + CelEvaluationException e = + assertThrows(CelEvaluationException.class, () -> cel.createProgram(ast).eval()); + assertThat(e) + .hasMessageThat() + .contains( + "evaluation error: Regex pattern exceeds allowed program size. Allowed: 6, Provided:" + + " 7"); + assertThat(e.getErrorCode()).isEqualTo(CelErrorCode.INVALID_ARGUMENT); + } + @Test public void toBuilder_isImmutable() { CelBuilder celBuilder = CelFactory.standardCelBuilder(); diff --git a/common/BUILD.bazel b/common/BUILD.bazel index af0bf8abb..78ab669bd 100644 --- a/common/BUILD.bazel +++ b/common/BUILD.bazel @@ -6,8 +6,13 @@ package( ) java_library( + # TODO: Split this target and migrate consumers name = "common", - exports = ["//common/src/main/java/dev/cel/common"], + exports = [ + "//common/src/main/java/dev/cel/common", + "//common/src/main/java/dev/cel/common:cel_ast", + "//common/src/main/java/dev/cel/common:cel_source", + ], ) java_library( @@ -72,3 +77,13 @@ java_library( name = "source_location", exports = ["//common/src/main/java/dev/cel/common:source_location"], ) + +java_library( + name = "cel_source", + exports = ["//common/src/main/java/dev/cel/common:cel_source"], +) + +java_library( + name = "cel_ast", + exports = ["//common/src/main/java/dev/cel/common:cel_ast"], +) diff --git a/common/src/main/java/dev/cel/common/BUILD.bazel b/common/src/main/java/dev/cel/common/BUILD.bazel index 31fd401c5..0592bce6d 100644 --- a/common/src/main/java/dev/cel/common/BUILD.bazel +++ b/common/src/main/java/dev/cel/common/BUILD.bazel @@ -10,11 +10,9 @@ package( # keep sorted COMMON_SOURCES = [ - "CelAbstractSyntaxTree.java", "CelDescriptorUtil.java", "CelDescriptors.java", "CelException.java", - "CelSource.java", ] # keep sorted @@ -49,6 +47,8 @@ java_library( tags = [ ], deps = [ + ":cel_ast", + ":cel_source", ":error_codes", ":source", ":source_location", @@ -72,6 +72,8 @@ java_library( tags = [ ], deps = [ + ":cel_ast", + ":cel_source", ":common", ":source", ":source_location", @@ -116,6 +118,8 @@ java_library( tags = [ ], deps = [ + ":cel_ast", + ":cel_source", "//common", "//common/ast:expr_converter", "//common/types:cel_proto_types", @@ -131,7 +135,10 @@ java_library( tags = [ ], deps = [ + ":cel_ast", + ":cel_source", ":common", + "//common:cel_source", "//common/ast:expr_v1alpha1_converter", "//common/types:cel_v1alpha1_types", "@com_google_googleapis//google/api/expr/v1alpha1:expr_java_proto", @@ -164,6 +171,7 @@ java_library( tags = [ ], deps = [ + ":cel_ast", ":mutable_source", "//common", "//common/ast", @@ -178,6 +186,7 @@ java_library( tags = [ ], deps = [ + ":cel_source", ":common", "//:auto_value", "//common/ast:mutable_expr", @@ -213,6 +222,44 @@ java_library( ], ) +java_library( + name = "cel_source", + srcs = ["CelSource.java"], + tags = [ + "alt_dep=//common:cel_source", + "avoid_dep", + ], + deps = [ + ":source", + ":source_location", + "//:auto_value", + "//common/annotations", + "//common/ast", + "//common/internal", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "cel_ast", + srcs = ["CelAbstractSyntaxTree.java"], + tags = [ + "alt_dep=//common:cel_ast", + "avoid_dep", + ], + deps = [ + ":cel_source", + "//:auto_value", + "//common/annotations", + "//common/ast", + "//common/types", + "//common/types:type_providers", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + java_library( name = "source", srcs = SOURCE_SOURCES, diff --git a/common/src/main/java/dev/cel/common/CelOptions.java b/common/src/main/java/dev/cel/common/CelOptions.java index 3e841471f..7c968ebb2 100644 --- a/common/src/main/java/dev/cel/common/CelOptions.java +++ b/common/src/main/java/dev/cel/common/CelOptions.java @@ -39,7 +39,7 @@ public enum ProtoUnsetFieldOptions { // Do not bind a field if it is unset. Repeated fields are bound as empty list. SKIP, // Bind the (proto api) default value for a field. - BIND_DEFAULT; + BIND_DEFAULT } public static final CelOptions DEFAULT = current().build(); @@ -69,6 +69,8 @@ public enum ProtoUnsetFieldOptions { public abstract boolean enableHiddenAccumulatorVar(); + public abstract boolean enableQuotedIdentifierSyntax(); + // Type-Checker related options public abstract boolean enableCompileTimeOverloadResolution(); @@ -119,6 +121,8 @@ public enum ProtoUnsetFieldOptions { public abstract boolean enableComprehension(); + public abstract int maxRegexProgramSize(); + public abstract Builder toBuilder(); public ImmutableSet toExprFeatures() { @@ -191,6 +195,7 @@ public static Builder newBuilder() { .retainRepeatedUnaryOperators(false) .retainUnbalancedLogicalExpressions(false) .enableHiddenAccumulatorVar(false) + .enableQuotedIdentifierSyntax(false) // Type-Checker options .enableCompileTimeOverloadResolution(false) .enableHomogeneousLiterals(false) @@ -215,7 +220,8 @@ public static Builder newBuilder() { .enableStringConversion(true) .enableStringConcatenation(true) .enableListConcatenation(true) - .enableComprehension(true); + .enableComprehension(true) + .maxRegexProgramSize(-1); } /** @@ -332,6 +338,16 @@ public abstract static class Builder { */ public abstract Builder enableHiddenAccumulatorVar(boolean value); + /** + * Enable quoted identifier syntax. + * + *

This enables the use of quoted identifier syntax when parsing CEL expressions. When + * enabled, the parser will accept identifiers that are surrounded by backticks (`) and will + * treat them as a single identifier. Currently, this is only supported for field specifiers + * over a limited character set. + */ + public abstract Builder enableQuotedIdentifierSyntax(boolean value); + // Type-Checker related options /** @@ -558,6 +574,19 @@ public abstract static class Builder { */ public abstract Builder enableComprehension(boolean value); + /** + * Set maximum program size for RE2J regex. + * + *

The program size is a very approximate measure of a regexp's "cost". Larger numbers are + * more expensive than smaller numbers. + * + *

A negative {@code value} will disable the check. + * + *

There's no guarantee that RE2 program size has the exact same value across other CEL + * implementations (C++ and Go). + */ + public abstract Builder maxRegexProgramSize(int value); + public abstract CelOptions build(); } } diff --git a/common/src/main/java/dev/cel/common/types/CelKind.java b/common/src/main/java/dev/cel/common/types/CelKind.java index a97fe5b55..7d55ddaf1 100644 --- a/common/src/main/java/dev/cel/common/types/CelKind.java +++ b/common/src/main/java/dev/cel/common/types/CelKind.java @@ -32,7 +32,6 @@ public enum CelKind { BYTES, DOUBLE, DURATION, - FUNCTION, INT, LIST, MAP, diff --git a/common/src/test/java/dev/cel/common/types/CelKindTest.java b/common/src/test/java/dev/cel/common/types/CelKindTest.java index a29a3a1e3..68b487afa 100644 --- a/common/src/test/java/dev/cel/common/types/CelKindTest.java +++ b/common/src/test/java/dev/cel/common/types/CelKindTest.java @@ -40,7 +40,6 @@ public void isPrimitive_false() { assertThat(CelKind.DYN.isPrimitive()).isFalse(); assertThat(CelKind.ANY.isPrimitive()).isFalse(); assertThat(CelKind.DURATION.isPrimitive()).isFalse(); - assertThat(CelKind.FUNCTION.isPrimitive()).isFalse(); assertThat(CelKind.LIST.isPrimitive()).isFalse(); assertThat(CelKind.MAP.isPrimitive()).isFalse(); assertThat(CelKind.NULL_TYPE.isPrimitive()).isFalse(); diff --git a/parser/src/main/java/dev/cel/parser/BUILD.bazel b/parser/src/main/java/dev/cel/parser/BUILD.bazel index 46f68b0bd..7203876e4 100644 --- a/parser/src/main/java/dev/cel/parser/BUILD.bazel +++ b/parser/src/main/java/dev/cel/parser/BUILD.bazel @@ -127,6 +127,8 @@ java_library( "//common", "//common/ast", "//common/ast:cel_expr_visitor", + "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_re2j_re2j", ], ) diff --git a/parser/src/main/java/dev/cel/parser/CelUnparserVisitor.java b/parser/src/main/java/dev/cel/parser/CelUnparserVisitor.java index 5f2f05e4d..79a6147c5 100644 --- a/parser/src/main/java/dev/cel/parser/CelUnparserVisitor.java +++ b/parser/src/main/java/dev/cel/parser/CelUnparserVisitor.java @@ -13,7 +13,9 @@ // limitations under the License. package dev.cel.parser; +import com.google.common.collect.ImmutableSet; import com.google.protobuf.ByteString; +import com.google.re2j.Pattern; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelSource; import dev.cel.common.ast.CelConstant; @@ -43,6 +45,10 @@ public class CelUnparserVisitor extends CelExprVisitor { protected static final String RIGHT_BRACE = "}"; protected static final String COLON = ":"; protected static final String QUESTION_MARK = "?"; + protected static final String BACKTICK = "`"; + private static final Pattern IDENTIFIER_SEGMENT_PATTERN = + Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*"); + private static final ImmutableSet RESTRICTED_FIELD_NAMES = ImmutableSet.of("in"); protected final CelAbstractSyntaxTree ast; protected final CelSource sourceInfo; @@ -60,6 +66,14 @@ public String unparse() { return stringBuilder.toString(); } + private static String maybeQuoteField(String field) { + if (RESTRICTED_FIELD_NAMES.contains(field) + || !IDENTIFIER_SEGMENT_PATTERN.matcher(field).matches()) { + return BACKTICK + field + BACKTICK; + } + return field; + } + @Override public void visit(CelExpr expr) { if (sourceInfo.getMacroCalls().containsKey(expr.id())) { @@ -191,7 +205,7 @@ protected void visit(CelExpr expr, CelStruct struct) { if (e.optionalEntry()) { stringBuilder.append(QUESTION_MARK); } - stringBuilder.append(e.fieldKey()); + stringBuilder.append(maybeQuoteField(e.fieldKey())); stringBuilder.append(COLON).append(SPACE); visit(e.value()); } @@ -263,7 +277,7 @@ private void visitSelect(CelExpr operand, boolean testOnly, String op, String fi } boolean nested = !testOnly && isBinaryOrTernaryOperator(operand); visitMaybeNested(operand, nested); - stringBuilder.append(op).append(field); + stringBuilder.append(op).append(maybeQuoteField(field)); if (testOnly) { stringBuilder.append(RIGHT_PAREN); } diff --git a/parser/src/main/java/dev/cel/parser/Parser.java b/parser/src/main/java/dev/cel/parser/Parser.java index b49f81453..2e7b08dc0 100644 --- a/parser/src/main/java/dev/cel/parser/Parser.java +++ b/parser/src/main/java/dev/cel/parser/Parser.java @@ -33,10 +33,13 @@ import cel.parser.internal.CELParser.CreateMapContext; import cel.parser.internal.CELParser.CreateMessageContext; import cel.parser.internal.CELParser.DoubleContext; +import cel.parser.internal.CELParser.EscapeIdentContext; +import cel.parser.internal.CELParser.EscapedIdentifierContext; import cel.parser.internal.CELParser.ExprContext; import cel.parser.internal.CELParser.ExprListContext; import cel.parser.internal.CELParser.FieldInitializerListContext; -import cel.parser.internal.CELParser.IdentOrGlobalCallContext; +import cel.parser.internal.CELParser.GlobalCallContext; +import cel.parser.internal.CELParser.IdentContext; import cel.parser.internal.CELParser.IndexContext; import cel.parser.internal.CELParser.IntContext; import cel.parser.internal.CELParser.ListInitContext; @@ -52,6 +55,7 @@ import cel.parser.internal.CELParser.PrimaryExprContext; import cel.parser.internal.CELParser.RelationContext; import cel.parser.internal.CELParser.SelectContext; +import cel.parser.internal.CELParser.SimpleIdentifierContext; import cel.parser.internal.CELParser.StartContext; import cel.parser.internal.CELParser.StringContext; import cel.parser.internal.CELParser.UintContext; @@ -438,7 +442,7 @@ public CelExpr visitSelect(SelectContext context) { if (context.id == null) { return exprFactory.newExprBuilder(context).build(); } - String id = context.id.getText(); + String id = normalizeEscapedIdent(context.id); if (context.opt != null && context.opt.getText().equals("?")) { if (!options.enableOptionalSyntax()) { @@ -535,7 +539,7 @@ public CelExpr visitCreateMessage(CreateMessageContext context) { } @Override - public CelExpr visitIdentOrGlobalCall(IdentOrGlobalCallContext context) { + public CelExpr visitIdent(IdentContext context) { checkNotNull(context); if (context.id == null) { return exprFactory.newExprBuilder(context).build(); @@ -547,11 +551,25 @@ public CelExpr visitIdentOrGlobalCall(IdentOrGlobalCallContext context) { if (context.leadingDot != null) { id = "." + id; } - if (context.op == null) { - return exprFactory - .newExprBuilder(context.id) - .setIdent(CelExpr.CelIdent.newBuilder().setName(id).build()) - .build(); + + return exprFactory + .newExprBuilder(context.id) + .setIdent(CelExpr.CelIdent.newBuilder().setName(id).build()) + .build(); + } + + @Override + public CelExpr visitGlobalCall(GlobalCallContext context) { + checkNotNull(context); + if (context.id == null) { + return exprFactory.newExprBuilder(context).build(); + } + String id = context.id.getText(); + if (options.enableReservedIds() && RESERVED_IDS.contains(id)) { + return exprFactory.reportError(context, "reserved identifier: %s", id); + } + if (context.leadingDot != null) { + id = "." + id; } return globalCallOrMacro(context, id); @@ -671,6 +689,24 @@ private Optional visitMacro( return expandedMacro; } + private String normalizeEscapedIdent(EscapeIdentContext context) { + String identifier = context.getText(); + if (context instanceof SimpleIdentifierContext) { + return identifier; + } else if (context instanceof EscapedIdentifierContext) { + if (!options.enableQuotedIdentifierSyntax()) { + exprFactory.reportError(context, "unsupported syntax '`'"); + return identifier; + } + return identifier.substring(1, identifier.length() - 1); + } + + // This is normally unreachable, but might happen if the parser is in an error state or if the + // grammar is updated and not handled here. + exprFactory.reportError(context, "unsupported identifier"); + return identifier; + } + private CelExpr.CelStruct.Builder visitStructFields(FieldInitializerListContext context) { if (context == null || context.cols == null @@ -692,10 +728,10 @@ private CelExpr.CelStruct.Builder visitStructFields(FieldInitializerListContext } // The field may be empty due to a prior error. - if (fieldContext.IDENTIFIER() == null) { + if (fieldContext.escapeIdent() == null) { return CelExpr.CelStruct.newBuilder(); } - String fieldName = fieldContext.IDENTIFIER().getText(); + String fieldName = normalizeEscapedIdent(fieldContext.escapeIdent()); CelExpr.CelStruct.Entry.Builder exprBuilder = CelExpr.CelStruct.Entry.newBuilder() @@ -872,7 +908,7 @@ private CelExpr receiverCallOrMacro(MemberCallContext context, String id, CelExp return macroOrCall(context.args, context.open, id, Optional.of(member), true); } - private CelExpr globalCallOrMacro(IdentOrGlobalCallContext context, String id) { + private CelExpr globalCallOrMacro(GlobalCallContext context, String id) { return macroOrCall(context.args, context.op, id, Optional.empty(), false); } diff --git a/parser/src/main/java/dev/cel/parser/gen/CEL.g4 b/parser/src/main/java/dev/cel/parser/gen/CEL.g4 index fbbd1b434..65f4b830d 100644 --- a/parser/src/main/java/dev/cel/parser/gen/CEL.g4 +++ b/parser/src/main/java/dev/cel/parser/gen/CEL.g4 @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + grammar CEL; // Grammar Rules @@ -44,26 +45,27 @@ calc ; unary - : member # MemberExpr - | (ops+='!')+ member # LogicalNot - | (ops+='-')+ member # Negate + : member # MemberExpr + | (ops+='!')+ member # LogicalNot + | (ops+='-')+ member # Negate ; member - : primary # PrimaryExpr - | member op='.' (opt='?')? id=IDENTIFIER # Select - | member op='.' id=IDENTIFIER open='(' args=exprList? ')' # MemberCall - | member op='[' (opt='?')? index=expr ']' # Index + : primary # PrimaryExpr + | member op='.' (opt='?')? id=escapeIdent # Select + | member op='.' id=IDENTIFIER open='(' args=exprList? ')' # MemberCall + | member op='[' (opt='?')? index=expr ']' # Index ; primary - : leadingDot='.'? id=IDENTIFIER (op='(' args=exprList? ')')? # IdentOrGlobalCall - | '(' e=expr ')' # Nested - | op='[' elems=listInit? ','? ']' # CreateList - | op='{' entries=mapInitializerList? ','? '}' # CreateMap + : leadingDot='.'? id=IDENTIFIER # Ident + | leadingDot='.'? id=IDENTIFIER (op='(' args=exprList? ')') # GlobalCall + | '(' e=expr ')' # Nested + | op='[' elems=listInit? ','? ']' # CreateList + | op='{' entries=mapInitializerList? ','? '}' # CreateMap | leadingDot='.'? ids+=IDENTIFIER (ops+='.' ids+=IDENTIFIER)* - op='{' entries=fieldInitializerList? ','? '}' # CreateMessage - | literal # ConstantLiteral + op='{' entries=fieldInitializerList? ','? '}' # CreateMessage + | literal # ConstantLiteral ; exprList @@ -79,13 +81,18 @@ fieldInitializerList ; optField - : (opt='?')? IDENTIFIER + : (opt='?')? escapeIdent ; mapInitializerList : keys+=optExpr cols+=':' values+=expr (',' keys+=optExpr cols+=':' values+=expr)* ; +escapeIdent + : id=IDENTIFIER # SimpleIdentifier + | id=ESC_IDENTIFIER # EscapedIdentifier + ; + optExpr : (opt='?')? e=expr ; @@ -197,3 +204,4 @@ STRING BYTES : ('b' | 'B') STRING; IDENTIFIER : (LETTER | '_') ( LETTER | DIGIT | '_')*; +ESC_IDENTIFIER : '`' (LETTER | DIGIT | '_' | '.' | '-' | '/' | ' ')+ '`'; diff --git a/parser/src/test/java/dev/cel/parser/CelParserParameterizedTest.java b/parser/src/test/java/dev/cel/parser/CelParserParameterizedTest.java index 22d40117c..1a83c3127 100644 --- a/parser/src/test/java/dev/cel/parser/CelParserParameterizedTest.java +++ b/parser/src/test/java/dev/cel/parser/CelParserParameterizedTest.java @@ -205,6 +205,18 @@ public void parser() { .setOptions(CelOptions.current().enableReservedIds(false).build()) .build(), "while"); + CelParser parserWithQuotedFields = + CelParserImpl.newBuilder() + .setOptions(CelOptions.current().enableQuotedIdentifierSyntax(true).build()) + .build(); + runTest(parserWithQuotedFields, "foo.`bar`"); + runTest(parserWithQuotedFields, "foo.`bar-baz`"); + runTest(parserWithQuotedFields, "foo.`bar baz`"); + runTest(parserWithQuotedFields, "foo.`bar.baz`"); + runTest(parserWithQuotedFields, "foo.`bar/baz`"); + runTest(parserWithQuotedFields, "foo.`bar_baz`"); + runTest(parserWithQuotedFields, "foo.`in`"); + runTest(parserWithQuotedFields, "Struct{`in`: false}"); } @Test @@ -273,6 +285,23 @@ public void parser_errors() { runTest(parserWithoutOptionalSupport, "a.?b && a[?b]"); runTest(parserWithoutOptionalSupport, "Msg{?field: value} && {?'key': value}"); runTest(parserWithoutOptionalSupport, "[?a, ?b]"); + + CelParser parserWithQuotedFields = + CelParserImpl.newBuilder() + .setOptions(CelOptions.current().enableQuotedIdentifierSyntax(true).build()) + .build(); + runTest(parserWithQuotedFields, "`bar`"); + runTest(parserWithQuotedFields, "foo.``"); + runTest(parserWithQuotedFields, "foo.`$bar`"); + + CelParser parserWithoutQuotedFields = + CelParserImpl.newBuilder() + .setStandardMacros(CelStandardMacro.HAS) + .setOptions(CelOptions.current().enableQuotedIdentifierSyntax(false).build()) + .build(); + runTest(parserWithoutQuotedFields, "foo.`bar`"); + runTest(parserWithoutQuotedFields, "Struct{`bar`: false}"); + runTest(parserWithoutQuotedFields, "has(.`.`"); } @Test diff --git a/parser/src/test/java/dev/cel/parser/CelUnparserImplTest.java b/parser/src/test/java/dev/cel/parser/CelUnparserImplTest.java index 9ef729d94..0f2c00022 100644 --- a/parser/src/test/java/dev/cel/parser/CelUnparserImplTest.java +++ b/parser/src/test/java/dev/cel/parser/CelUnparserImplTest.java @@ -39,7 +39,11 @@ public final class CelUnparserImplTest { private final CelParser parser = CelParserImpl.newBuilder() - .setOptions(CelOptions.newBuilder().populateMacroCalls(true).build()) + .setOptions( + CelOptions.newBuilder() + .enableQuotedIdentifierSyntax(true) + .populateMacroCalls(true) + .build()) .addLibraries(CelOptionalLibrary.INSTANCE) .setStandardMacros(CelStandardMacro.STANDARD_MACROS) .build(); @@ -99,6 +103,15 @@ public List provideValues(Context context) { "a ? (b1 || b2) : (c1 && c2)", "(a ? b : c).method(d)", "a + b + c + d", + "foo.`a.b`", + "foo.`a/b`", + "foo.`a-b`", + "foo.`a b`", + "foo.`in`", + "Foo{`a.b`: foo}", + "Foo{`a/b`: foo}", + "Foo{`a-b`: foo}", + "Foo{`a b`: foo}", // Constants "true", @@ -140,6 +153,7 @@ public List provideValues(Context context) { // Macros "has(x[\"a\"].single_int32)", + "has(x.`foo-bar`.single_int32)", // This is a filter expression but is decompiled back to // map(x, filter_function, x) for which the evaluation is diff --git a/parser/src/test/resources/parser.baseline b/parser/src/test/resources/parser.baseline index 435b92a1f..9b509f61e 100644 --- a/parser/src/test/resources/parser.baseline +++ b/parser/src/test/resources/parser.baseline @@ -1622,4 +1622,48 @@ L: [ I: while =====> P: while^#1:Expr.Ident# -L: while^#1[1,0]# \ No newline at end of file +L: while^#1[1,0]# + +I: foo.`bar` +=====> +P: foo^#1:Expr.Ident#.bar^#2:Expr.Select# +L: foo^#1[1,0]#.bar^#2[1,3]# + +I: foo.`bar-baz` +=====> +P: foo^#1:Expr.Ident#.bar-baz^#2:Expr.Select# +L: foo^#1[1,0]#.bar-baz^#2[1,3]# + +I: foo.`bar baz` +=====> +P: foo^#1:Expr.Ident#.bar baz^#2:Expr.Select# +L: foo^#1[1,0]#.bar baz^#2[1,3]# + +I: foo.`bar.baz` +=====> +P: foo^#1:Expr.Ident#.bar.baz^#2:Expr.Select# +L: foo^#1[1,0]#.bar.baz^#2[1,3]# + +I: foo.`bar/baz` +=====> +P: foo^#1:Expr.Ident#.bar/baz^#2:Expr.Select# +L: foo^#1[1,0]#.bar/baz^#2[1,3]# + +I: foo.`bar_baz` +=====> +P: foo^#1:Expr.Ident#.bar_baz^#2:Expr.Select# +L: foo^#1[1,0]#.bar_baz^#2[1,3]# + +I: foo.`in` +=====> +P: foo^#1:Expr.Ident#.in^#2:Expr.Select# +L: foo^#1[1,0]#.in^#2[1,3]# + +I: Struct{`in`: false} +=====> +P: Struct{ + in:false^#3:bool#^#2:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: Struct{ + in:false^#3[1,13]#^#2[1,11]# +}^#1[1,6]# \ No newline at end of file diff --git a/parser/src/test/resources/parser_errors.baseline b/parser/src/test/resources/parser_errors.baseline index 8547ebed9..9f4b96825 100644 --- a/parser/src/test/resources/parser_errors.baseline +++ b/parser/src/test/resources/parser_errors.baseline @@ -255,7 +255,7 @@ E: ERROR: :1:2: mismatched input '' expecting {'[', '{', '}', '(', ' I: t{>C} =====> -E: ERROR: :1:3: extraneous input '>' expecting {'}', ',', '?', IDENTIFIER} +E: ERROR: :1:3: extraneous input '>' expecting {'}', ',', '?', IDENTIFIER, ESC_IDENTIFIER} | t{>C} | ..^ ERROR: :1:5: mismatched input '}' expecting ':' @@ -296,4 +296,52 @@ E: ERROR: :1:2: unsupported syntax '?' | .^ ERROR: :1:6: unsupported syntax '?' | [?a, ?b] - | .....^ \ No newline at end of file + | .....^ + +I: `bar` +=====> +E: ERROR: :1:1: mismatched input '`bar`' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} + | `bar` + | ^ + +I: foo.`` +=====> +E: ERROR: :1:5: token recognition error at: '``' + | foo.`` + | ....^ +ERROR: :1:7: no viable alternative at input '.' + | foo.`` + | ......^ + +I: foo.`$bar` +=====> +E: ERROR: :1:5: token recognition error at: '`$' + | foo.`$bar` + | ....^ +ERROR: :1:10: token recognition error at: '`' + | foo.`$bar` + | .........^ + +I: foo.`bar` +=====> +E: ERROR: :1:5: unsupported syntax '`' + | foo.`bar` + | ....^ + +I: Struct{`bar`: false} +=====> +E: ERROR: :1:8: unsupported syntax '`' + | Struct{`bar`: false} + | .......^ + +I: has(.`.` +=====> +E: ERROR: :1:6: no viable alternative at input '.`.`' + | has(.`.` + | .....^ +ERROR: :1:6: unsupported syntax '`' + | has(.`.` + | .....^ +ERROR: :1:9: missing ')' at '' + | has(.`.` + | ........^ diff --git a/policy/src/test/resources/compile_errors/expected_errors.baseline b/policy/src/test/resources/compile_errors/expected_errors.baseline index a8c0ec047..d03f27135 100644 --- a/policy/src/test/resources/compile_errors/expected_errors.baseline +++ b/policy/src/test/resources/compile_errors/expected_errors.baseline @@ -1,7 +1,7 @@ ERROR: compile_errors/policy.yaml:19:19: undeclared reference to 'spec' (in container '') | expression: spec.labels | ..................^ -ERROR: compile_errors/policy.yaml:21:50: mismatched input 'resource' expecting {'==', '!=', 'in', '<', '<=', '>=', '>', '&&', '||', '[', '(', ')', '.', '-', '?', '+', '*', '/', '%%'} +ERROR: compile_errors/policy.yaml:21:50: mismatched input 'resource' expecting {'==', '!=', 'in', '<', '<=', '>=', '>', '&&', '||', '[', ')', '.', '-', '?', '+', '*', '/', '%%'} | expression: variables.want.filter(l, !(lin resource.labels)) | .................................................^ ERROR: compile_errors/policy.yaml:21:66: extraneous input ')' expecting diff --git a/publish/BUILD.bazel b/publish/BUILD.bazel index 050292eab..1d0530077 100644 --- a/publish/BUILD.bazel +++ b/publish/BUILD.bazel @@ -47,10 +47,14 @@ OPTIMIZER_TARGETS = [ "//optimizer/src/main/java/dev/cel/optimizer/optimizers:common_subexpression_elimination", ] -V1ALPHA1_UTILITY_TARGETS = [ +V1ALPHA1_AST_TARGETS = [ "//common/src/main/java/dev/cel/common:proto_v1alpha1_ast", ] +CANONICAL_AST_TARGETS = [ + "//common/src/main/java/dev/cel/common:proto_ast", +] + EXTENSION_TARGETS = [ "//extensions/src/main/java/dev/cel/extensions", "//extensions/src/main/java/dev/cel/extensions:optional_library", @@ -58,7 +62,12 @@ EXTENSION_TARGETS = [ ALL_TARGETS = [ "//bundle/src/main/java/dev/cel/bundle:cel", -] + RUNTIME_TARGETS + COMPILER_TARGETS + EXTENSION_TARGETS + V1ALPHA1_UTILITY_TARGETS + OPTIMIZER_TARGETS + VALIDATOR_TARGETS +] + RUNTIME_TARGETS + COMPILER_TARGETS + EXTENSION_TARGETS + V1ALPHA1_AST_TARGETS + CANONICAL_AST_TARGETS + OPTIMIZER_TARGETS + VALIDATOR_TARGETS + +# Excluded from the JAR as their source of truth is elsewhere +EXCLUDED_TARGETS = [ + "@com_google_googleapis//google/api/expr/v1alpha1:expr_java_proto", +] pom_file( name = "cel_pom", @@ -74,9 +83,7 @@ pom_file( java_export( name = "cel", - deploy_env = [ - "@com_google_googleapis//google/api/expr/v1alpha1:expr_java_proto", - ], + deploy_env = EXCLUDED_TARGETS, maven_coordinates = "dev.cel:cel:%s" % CEL_VERSION, pom_template = ":cel_pom", runtime_deps = ALL_TARGETS, @@ -96,9 +103,7 @@ pom_file( java_export( name = "cel_compiler", - deploy_env = [ - "@com_google_googleapis//google/api/expr/v1alpha1:expr_java_proto", - ], + deploy_env = EXCLUDED_TARGETS, maven_coordinates = "dev.cel:compiler:%s" % CEL_VERSION, pom_template = ":cel_compiler_pom", runtime_deps = COMPILER_TARGETS, @@ -118,9 +123,7 @@ pom_file( java_export( name = "cel_runtime", - deploy_env = [ - "@com_google_googleapis//google/api/expr/v1alpha1:expr_java_proto", - ], + deploy_env = EXCLUDED_TARGETS, maven_coordinates = "dev.cel:runtime:%s" % CEL_VERSION, pom_template = ":cel_runtime_pom", runtime_deps = RUNTIME_TARGETS, @@ -134,16 +137,34 @@ pom_file( "PACKAGE_NAME": "CEL Java v1alpha1 Utility", "PACKAGE_DESC": "Common Expression Language Utility for supporting v1alpha1 protobuf definitions", }, - targets = V1ALPHA1_UTILITY_TARGETS, + targets = V1ALPHA1_AST_TARGETS, template_file = "pom_template.xml", ) java_export( name = "cel_v1alpha1", - deploy_env = [ - "@com_google_googleapis//google/api/expr/v1alpha1:expr_java_proto", - ], + deploy_env = EXCLUDED_TARGETS, maven_coordinates = "dev.cel:v1alpha1:%s" % CEL_VERSION, pom_template = ":cel_v1alpha1_pom", - runtime_deps = V1ALPHA1_UTILITY_TARGETS, + runtime_deps = V1ALPHA1_AST_TARGETS, +) + +pom_file( + name = "cel_protobuf_pom", + substitutions = { + "CEL_VERSION": CEL_VERSION, + "CEL_ARTIFACT_ID": "protobuf", + "PACKAGE_NAME": "CEL Java Protobuf adapter", + "PACKAGE_DESC": "Common Expression Language Adapter for converting canonical cel.expr protobuf definitions", + }, + targets = CANONICAL_AST_TARGETS, + template_file = "pom_template.xml", +) + +java_export( + name = "cel_protobuf", + deploy_env = EXCLUDED_TARGETS, + maven_coordinates = "dev.cel:protobuf:%s" % CEL_VERSION, + pom_template = ":cel_protobuf_pom", + runtime_deps = CANONICAL_AST_TARGETS, ) diff --git a/publish/cel_version.bzl b/publish/cel_version.bzl index a0ebb4dd7..1847a1923 100644 --- a/publish/cel_version.bzl +++ b/publish/cel_version.bzl @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. """Maven artifact version for CEL.""" -CEL_VERSION = "0.9.0" +CEL_VERSION = "0.9.1" diff --git a/publish/publish.sh b/publish/publish.sh index b5d301ce2..8a284ba48 100755 --- a/publish/publish.sh +++ b/publish/publish.sh @@ -25,7 +25,7 @@ # 2. You will need to enter the key's password. The prompt appears in GUI, not in terminal. The publish operation will eventually timeout if the password is not entered. -ALL_TARGETS=("//publish:cel.publish" "//publish:cel_compiler.publish" "//publish:cel_runtime.publish" "//publish:cel_v1alpha1.publish") +ALL_TARGETS=("//publish:cel.publish" "//publish:cel_compiler.publish" "//publish:cel_runtime.publish" "//publish:cel_v1alpha1.publish" "//publish:cel_protobuf.publish") function publish_maven_remote() { maven_repo_url=$1 diff --git a/runtime/BUILD.bazel b/runtime/BUILD.bazel index 3635c0961..7058bfc8b 100644 --- a/runtime/BUILD.bazel +++ b/runtime/BUILD.bazel @@ -12,7 +12,6 @@ java_library( java_library( name = "interpreter", - visibility = ["//visibility:public"], exports = ["//runtime/src/main/java/dev/cel/runtime:interpreter"], ) @@ -41,3 +40,8 @@ java_library( name = "evaluation_listener", exports = ["//runtime/src/main/java/dev/cel/runtime:evaluation_listener"], ) + +java_library( + name = "base", + exports = ["//runtime/src/main/java/dev/cel/runtime:base"], +) diff --git a/runtime/src/main/java/dev/cel/runtime/CelStandardFunctions.java b/runtime/src/main/java/dev/cel/runtime/CelStandardFunctions.java index bd1fc45f9..dc6ca979b 100644 --- a/runtime/src/main/java/dev/cel/runtime/CelStandardFunctions.java +++ b/runtime/src/main/java/dev/cel/runtime/CelStandardFunctions.java @@ -28,7 +28,6 @@ import com.google.protobuf.Timestamp; import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; -import com.google.re2j.PatternSyntaxException; import dev.cel.common.CelErrorCode; import dev.cel.common.CelOptions; import dev.cel.common.internal.ComparisonFunctions; @@ -1000,7 +999,7 @@ public enum StringMatchers implements StandardOverload { (String string, String regexp) -> { try { return RuntimeHelpers.matches(string, regexp, bindingHelper.celOptions); - } catch (PatternSyntaxException e) { + } catch (RuntimeException e) { throw new CelEvaluationException( e.getMessage(), e, CelErrorCode.INVALID_ARGUMENT); } @@ -1015,7 +1014,7 @@ public enum StringMatchers implements StandardOverload { (String string, String regexp) -> { try { return RuntimeHelpers.matches(string, regexp, bindingHelper.celOptions); - } catch (PatternSyntaxException e) { + } catch (RuntimeException e) { throw new CelEvaluationException( e.getMessage(), e, CelErrorCode.INVALID_ARGUMENT); } diff --git a/runtime/src/main/java/dev/cel/runtime/RuntimeHelpers.java b/runtime/src/main/java/dev/cel/runtime/RuntimeHelpers.java index ffb979842..b885bb55d 100644 --- a/runtime/src/main/java/dev/cel/runtime/RuntimeHelpers.java +++ b/runtime/src/main/java/dev/cel/runtime/RuntimeHelpers.java @@ -74,12 +74,22 @@ public static boolean matches(String string, String regexp) { } public static boolean matches(String string, String regexp, CelOptions celOptions) { + Pattern pattern = Pattern.compile(regexp); + int maxProgramSize = celOptions.maxRegexProgramSize(); + if (maxProgramSize >= 0 && pattern.programSize() > maxProgramSize) { + throw new IllegalArgumentException( + String.format( + "Regex pattern exceeds allowed program size. Allowed: %d, Provided: %d", + maxProgramSize, pattern.programSize())); + } + if (!celOptions.enableRegexPartialMatch()) { // Uses re2 for consistency across languages. - return Pattern.matches(regexp, string); + return pattern.matcher(string).matches(); } - // Return an unanchored match for the presence of the regexp anywher in the string. - return Pattern.compile(regexp).matcher(string).find(); + + // Return an unanchored match for the presence of the regexp anywhere in the string. + return pattern.matcher(string).find(); } /** Create a compiled pattern for the given regular expression. */